From d6e11cf0181ecd802a085af58f2afaeb2b88a6f3 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Tue, 4 Aug 2026 11:48:00 +0800 Subject: [PATCH] refactor(table-catalog): modularize catalog implementation (#5678) * refactor(table-catalog): split catalog foundations * refactor(table-catalog): split REST handler modules * refactor(table-catalog): split domain and store modules --------- Co-authored-by: Henry Guo --- rustfs/src/admin/handlers/table_catalog.rs | 12863 ---------- .../admin/handlers/table_catalog/config.rs | 188 + .../handlers/table_catalog/credentials.rs | 34 + .../handlers/table_catalog/maintenance.rs | 223 + .../src/admin/handlers/table_catalog/mod.rs | 4997 ++++ .../admin/handlers/table_catalog/namespace.rs | 93 + .../src/admin/handlers/table_catalog/refs.rs | 75 + .../admin/handlers/table_catalog/routes.rs | 259 + .../src/admin/handlers/table_catalog/table.rs | 296 + .../src/admin/handlers/table_catalog/tests.rs | 6712 +++++ .../src/admin/handlers/table_catalog/view.rs | 120 + rustfs/src/table_catalog.rs | 20973 ---------------- rustfs/src/table_catalog/error.rs | 79 + rustfs/src/table_catalog/iceberg/commit.rs | 303 + rustfs/src/table_catalog/iceberg/manifest.rs | 177 + rustfs/src/table_catalog/iceberg/metadata.rs | 120 + rustfs/src/table_catalog/iceberg/mod.rs | 23 + .../src/table_catalog/iceberg/validation.rs | 219 + rustfs/src/table_catalog/identifier.rs | 355 + .../src/table_catalog/maintenance/config.rs | 113 + rustfs/src/table_catalog/maintenance/mod.rs | 23 + .../src/table_catalog/maintenance/planner.rs | 1421 ++ .../src/table_catalog/maintenance/recovery.rs | 716 + .../src/table_catalog/maintenance/worker.rs | 225 + rustfs/src/table_catalog/mod.rs | 345 + rustfs/src/table_catalog/model.rs | 1353 + rustfs/src/table_catalog/store/migration.rs | 881 + rustfs/src/table_catalog/store/mod.rs | 1173 + rustfs/src/table_catalog/store/object.rs | 3825 +++ rustfs/src/table_catalog/store/strong.rs | 1363 + rustfs/src/table_catalog/tests.rs | 8580 +++++++ 31 files changed, 34291 insertions(+), 33836 deletions(-) delete mode 100644 rustfs/src/admin/handlers/table_catalog.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/config.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/credentials.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/maintenance.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/mod.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/namespace.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/refs.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/routes.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/table.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/tests.rs create mode 100644 rustfs/src/admin/handlers/table_catalog/view.rs delete mode 100644 rustfs/src/table_catalog.rs create mode 100644 rustfs/src/table_catalog/error.rs create mode 100644 rustfs/src/table_catalog/iceberg/commit.rs create mode 100644 rustfs/src/table_catalog/iceberg/manifest.rs create mode 100644 rustfs/src/table_catalog/iceberg/metadata.rs create mode 100644 rustfs/src/table_catalog/iceberg/mod.rs create mode 100644 rustfs/src/table_catalog/iceberg/validation.rs create mode 100644 rustfs/src/table_catalog/identifier.rs create mode 100644 rustfs/src/table_catalog/maintenance/config.rs create mode 100644 rustfs/src/table_catalog/maintenance/mod.rs create mode 100644 rustfs/src/table_catalog/maintenance/planner.rs create mode 100644 rustfs/src/table_catalog/maintenance/recovery.rs create mode 100644 rustfs/src/table_catalog/maintenance/worker.rs create mode 100644 rustfs/src/table_catalog/mod.rs create mode 100644 rustfs/src/table_catalog/model.rs create mode 100644 rustfs/src/table_catalog/store/migration.rs create mode 100644 rustfs/src/table_catalog/store/mod.rs create mode 100644 rustfs/src/table_catalog/store/object.rs create mode 100644 rustfs/src/table_catalog/store/strong.rs create mode 100644 rustfs/src/table_catalog/tests.rs diff --git a/rustfs/src/admin/handlers/table_catalog.rs b/rustfs/src/admin/handlers/table_catalog.rs deleted file mode 100644 index b20b63d61..000000000 --- a/rustfs/src/admin/handlers/table_catalog.rs +++ /dev/null @@ -1,12863 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::admin::runtime_sources::default_admin_usecase; -use crate::admin::runtime_sources::{current_object_store_handle, current_token_signing_key}; -use crate::admin::storage_api::bucket::{metadata::table_catalog_path_hash, metadata_sys}; -use crate::admin::storage_api::runtime::ECStore; -use crate::admin::{ - auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object}, - router::{AdminOperation, Operation, S3Router}, -}; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{RemoteAddr, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX}; -use crate::table_catalog::{DEFAULT_WAREHOUSE_ID, TableCatalogStore}; -use http::{HeaderMap, HeaderValue, StatusCode}; -use hyper::Method; -use matchit::Params; -use metrics::{counter, histogram}; -use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; -use rustfs_iam::sys::SESSION_POLICY_NAME; -use rustfs_policy::{ - auth::get_new_credentials_with_metadata, - policy::{ - Policy, - action::{Action, AdminAction}, - }, -}; -use rustfs_utils::crypto::{base64_decode_url_safe_no_pad, base64_encode_url_safe_no_pad, hex_sha256}; -use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::num::NonZeroUsize; -use std::time::{Duration as StdDuration, Instant}; -use time::{Duration, OffsetDateTime}; -use uuid::Uuid; - -const JSON_CONTENT_TYPE: &str = "application/json"; -const ENV_TABLE_CATALOG_CREDENTIAL_VENDING: &str = "RUSTFS_TABLE_CATALOG_CREDENTIAL_VENDING"; -const ENV_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: &str = "RUSTFS_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS"; -const DEFAULT_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 15 * 60; -const MIN_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60; -const MAX_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60 * 60; -const WAREHOUSE_PROPERTY: &str = "warehouse"; -const PREFIX_PROPERTY: &str = "prefix"; -const ICEBERG_ERROR_ALREADY_EXISTS: &str = "AlreadyExistsException"; -const ICEBERG_ERROR_BAD_REQUEST: &str = "BadRequestException"; -const ICEBERG_ERROR_COMMIT_FAILED: &str = "CommitFailedException"; -const ICEBERG_ERROR_NAMESPACE_NOT_EMPTY: &str = "NamespaceNotEmptyException"; -const ICEBERG_ERROR_NO_SUCH_NAMESPACE: &str = "NoSuchNamespaceException"; -const ICEBERG_ERROR_NO_SUCH_RESOURCE: &str = "NoSuchResourceException"; -const ICEBERG_ERROR_NO_SUCH_TABLE: &str = "NoSuchTableException"; -const ICEBERG_ERROR_NO_SUCH_VIEW: &str = "NoSuchViewException"; -const ICEBERG_ERROR_REST: &str = "RESTException"; -const REST_PAGE_TOKEN_VERSION: u8 = 1; -const REST_PAGE_TOKEN_MAX_LENGTH: usize = 16 * 1024; -const REST_DEFAULT_PAGE_SIZE: usize = 1000; -const REST_MAX_PAGE_SIZE: usize = 1000; -const REST_PAGE_TOKEN_QUERY_PARAMETER: &str = "pageToken"; -const REST_PAGE_SIZE_QUERY_PARAMETER: &str = "pageSize"; -const CATALOG_ENDPOINT_PREFIX_CONFIG_KEY: &str = "rustfs.catalog-endpoint-prefix"; -const CATALOG_COMPAT_ENDPOINT_PREFIX_CONFIG_KEY: &str = "rustfs.catalog-compat-endpoint-prefix"; -const CATALOG_BACKING_CONFIG_KEY: &str = "rustfs.catalog-backing"; -const CREDENTIAL_VENDING_CONFIG_KEY: &str = "rustfs.credential-vending"; -const CREDENTIAL_VENDING_REASON_CONFIG_KEY: &str = "rustfs.credential-vending-reason"; -const CREDENTIAL_SCOPE_CONFIG_KEY: &str = "rustfs.credential-scope"; -const CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY: &str = "rustfs.credential-scope-prefix"; -const CREDENTIAL_MODE_CONFIG_KEY: &str = "rustfs.credential-mode"; -const CREDENTIAL_EXPIRATION_CONFIG_KEY: &str = "rustfs.credential-expiration-unix-seconds"; -const CREDENTIAL_VENDING_UNSUPPORTED: &str = "unsupported"; -const CREDENTIAL_VENDING_SUPPORTED: &str = "supported"; -const CREDENTIAL_VENDING_UNSUPPORTED_REASON: &str = "temporary-credentials-not-implemented"; -const CREDENTIAL_VENDING_DISABLED_REASON: &str = "credential-vending-disabled"; -const CREDENTIAL_SCOPE_WAREHOUSE_PREFIX: &str = "warehouse-prefix"; -const CREDENTIAL_SCOPE_TABLE_PREFIX: &str = "table-prefix"; -const CREDENTIAL_MODE_CLIENT_PROVIDED: &str = "client-provided-s3-credentials-required"; -const CREDENTIAL_MODE_CATALOG_VENDED: &str = "catalog-vended-temporary-credentials"; -const S3_ACCESS_KEY_ID_CONFIG_KEY: &str = "s3.access-key-id"; -const S3_SECRET_ACCESS_KEY_CONFIG_KEY: &str = "s3.secret-access-key"; -const S3_SESSION_TOKEN_CONFIG_KEY: &str = "s3.session-token"; -const TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT: &str = "namespaces"; -const TABLE_CATALOG_TABLE_RESOURCE_ROOT: &str = "tables"; -const TABLE_CATALOG_VIEW_RESOURCE_ROOT: &str = "views"; -const TABLE_CATALOG_ADMIN_OPERATION_SLOW_LOG_THRESHOLD: StdDuration = StdDuration::from_secs(2); -const DEFAULT_TABLE_MAINTENANCE_SCHEDULER_ID: &str = "rustfs-maintenance-scheduler"; -const DEFAULT_TABLE_MAINTENANCE_WORKER_ID: &str = "rustfs-maintenance-worker"; -const EXTERNAL_CATALOG_BRIDGE_STATUS_UNCONFIGURED: &str = "bridge-unconfigured"; -const EXTERNAL_CATALOG_BRIDGE_STATUS_CONFIGURED: &str = "bridge-configured"; -const EXTERNAL_CATALOG_BRIDGE_SYNC_STATUS: &str = "synced"; -const EXTERNAL_CATALOG_BRIDGE_SUPPORTED_STATUS: &str = "operator-sync-supported"; -const EXTERNAL_CATALOG_BRIDGE_SUPPORTED_REASON: &str = - "operator-supplied metadata pointer sync is supported; online vendor SDK synchronization is not claimed"; -const EXTERNAL_CATALOG_POLICY_MODE_RUSTFS: &str = "rustfs-authoritative"; -const EXTERNAL_CATALOG_CREDENTIAL_MODE_RUSTFS: &str = "rustfs-table-credentials"; -const EXTERNAL_CATALOG_SYNC_MODE_MANUAL: &str = "manual"; -const EXTERNAL_CATALOG_ROLLBACK_RETAIN_CURRENT: &str = "retain-current-pointer"; -const EXTERNAL_CATALOG_SYNC_OPERATION: &str = "external-catalog-sync"; -const EXTERNAL_CATALOG_SYNC_WRITER: &str = "rustfs-external-catalog-bridge"; -const EXTERNAL_CATALOG_ACTION_REGISTERED: &str = "registered"; -const EXTERNAL_CATALOG_ACTION_COMMITTED: &str = "committed"; -const EXTERNAL_CATALOG_BRIDGE_CAPABILITIES: &[&str] = &["polaris", "glue", "dlf", "hive-metastore"]; -const TABLE_CATALOG_ENDPOINTS: &[&str] = &[ - "GET /v1/{prefix}/namespaces", - "POST /v1/{prefix}/namespaces", - "GET /v1/{prefix}/namespaces/{namespace}", - "HEAD /v1/{prefix}/namespaces/{namespace}", - "DELETE /v1/{prefix}/namespaces/{namespace}", - "GET /v1/{prefix}/namespaces/{namespace}/tables", - "POST /v1/{prefix}/namespaces/{namespace}/tables", - "POST /v1/{prefix}/namespaces/{namespace}/register", - "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials", - "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "PUT /buckets/{warehouse}", - "GET /buckets/{warehouse}", - "GET /{warehouse}/catalog/migration", - "POST /{warehouse}/catalog/migration", - "DELETE /{warehouse}/catalog/migration", - "GET /{warehouse}/namespaces", - "POST /{warehouse}/namespaces", - "GET /{warehouse}/namespaces/{namespace}", - "HEAD /{warehouse}/namespaces/{namespace}", - "DELETE /{warehouse}/namespaces/{namespace}", - "GET /{warehouse}/namespaces/{namespace}/tables", - "POST /{warehouse}/namespaces/{namespace}/tables", - "POST /{warehouse}/namespaces/{namespace}/register", - "GET /{warehouse}/namespaces/{namespace}/views", - "POST /{warehouse}/namespaces/{namespace}/views", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}", - "HEAD /{warehouse}/namespaces/{namespace}/tables/{table}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/credentials", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}", - "DELETE /{warehouse}/namespaces/{namespace}/tables/{table}", - "GET /{warehouse}/namespaces/{namespace}/views/{view}", - "HEAD /{warehouse}/namespaces/{namespace}/views/{view}", - "POST /{warehouse}/namespaces/{namespace}/views/{view}", - "DELETE /{warehouse}/namespaces/{namespace}/views/{view}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/refs", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}", - "DELETE /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/heartbeat", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/quarantine", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external/sync", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/recovery", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback", -]; - -static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {}; -static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {}; -static GET_TABLE_BUCKET_HANDLER: GetTableBucketHandler = GetTableBucketHandler {}; -static GET_TABLE_CATALOG_MIGRATION_HANDLER: GetTableCatalogMigrationHandler = GetTableCatalogMigrationHandler {}; -static MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER: MaterializeTableCatalogMigrationHandler = - MaterializeTableCatalogMigrationHandler {}; -static CANCEL_TABLE_CATALOG_MIGRATION_HANDLER: CancelTableCatalogMigrationHandler = CancelTableCatalogMigrationHandler {}; -static LIST_NAMESPACES_HANDLER: RestListNamespacesHandler = RestListNamespacesHandler {}; -static CREATE_NAMESPACE_HANDLER: RestCreateNamespaceHandler = RestCreateNamespaceHandler {}; -static GET_NAMESPACE_HANDLER: RestGetNamespaceHandler = RestGetNamespaceHandler {}; -static NAMESPACE_EXISTS_HANDLER: RestNamespaceExistsHandler = RestNamespaceExistsHandler {}; -static DROP_NAMESPACE_HANDLER: RestDropNamespaceHandler = RestDropNamespaceHandler {}; -static LIST_TABLES_HANDLER: RestListTablesHandler = RestListTablesHandler {}; -static CREATE_TABLE_HANDLER: RestCreateTableHandler = RestCreateTableHandler {}; -static REGISTER_TABLE_HANDLER: RestRegisterTableHandler = RestRegisterTableHandler {}; -static LIST_VIEWS_HANDLER: RestListViewsHandler = RestListViewsHandler {}; -static CREATE_VIEW_HANDLER: RestCreateViewHandler = RestCreateViewHandler {}; -static LOAD_TABLE_HANDLER: RestLoadTableHandler = RestLoadTableHandler {}; -static TABLE_EXISTS_HANDLER: RestTableExistsHandler = RestTableExistsHandler {}; -static LOAD_CREDENTIALS_HANDLER: RestLoadCredentialsHandler = RestLoadCredentialsHandler {}; -static COMMIT_TABLE_HANDLER: RestCommitTableHandler = RestCommitTableHandler {}; -static DROP_TABLE_HANDLER: RestDropTableHandler = RestDropTableHandler {}; -static LOAD_VIEW_HANDLER: RestLoadViewHandler = RestLoadViewHandler {}; -static VIEW_EXISTS_HANDLER: RestViewExistsHandler = RestViewExistsHandler {}; -static REPLACE_VIEW_HANDLER: RestReplaceViewHandler = RestReplaceViewHandler {}; -static DROP_VIEW_HANDLER: RestDropViewHandler = RestDropViewHandler {}; -static LIST_TABLE_REFS_HANDLER: ListTableRefsHandler = ListTableRefsHandler {}; -static PUT_TABLE_REF_HANDLER: PutTableRefHandler = PutTableRefHandler {}; -static DELETE_TABLE_REF_HANDLER: DeleteTableRefHandler = DeleteTableRefHandler {}; -static GET_TABLE_METADATA_LOCATION_HANDLER: GetTableMetadataLocationHandler = GetTableMetadataLocationHandler {}; -static UPDATE_TABLE_METADATA_LOCATION_HANDLER: UpdateTableMetadataLocationHandler = UpdateTableMetadataLocationHandler {}; -static TABLE_METADATA_MAINTENANCE_HANDLER: RestTableMetadataMaintenanceHandler = RestTableMetadataMaintenanceHandler {}; -static GET_TABLE_MAINTENANCE_CONFIG_HANDLER: GetTableMaintenanceConfigHandler = GetTableMaintenanceConfigHandler {}; -static PUT_TABLE_MAINTENANCE_CONFIG_HANDLER: PutTableMaintenanceConfigHandler = PutTableMaintenanceConfigHandler {}; -static GET_TABLE_MAINTENANCE_JOB_HANDLER: GetTableMaintenanceJobHandler = GetTableMaintenanceJobHandler {}; -static GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER: GetTableMaintenanceSchedulerHandler = GetTableMaintenanceSchedulerHandler {}; -static RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER: RunTableMaintenanceSchedulerHandler = RunTableMaintenanceSchedulerHandler {}; -static RUN_TABLE_MAINTENANCE_WORKER_HANDLER: RunTableMaintenanceWorkerHandler = RunTableMaintenanceWorkerHandler {}; -static HEARTBEAT_TABLE_MAINTENANCE_JOB_HANDLER: HeartbeatTableMaintenanceJobHandler = HeartbeatTableMaintenanceJobHandler {}; -static TABLE_MAINTENANCE_QUARANTINE_HANDLER: TableMaintenanceQuarantineHandler = TableMaintenanceQuarantineHandler {}; -static EXPORT_TABLE_CATALOG_HANDLER: ExportTableCatalogHandler = ExportTableCatalogHandler {}; -static IMPORT_TABLE_CATALOG_HANDLER: ImportTableCatalogHandler = ImportTableCatalogHandler {}; -static EXTERNAL_CATALOG_BRIDGE_HANDLER: ExternalCatalogBridgeHandler = ExternalCatalogBridgeHandler {}; -static PUT_EXTERNAL_CATALOG_BRIDGE_HANDLER: PutExternalCatalogBridgeHandler = PutExternalCatalogBridgeHandler {}; -static SYNC_EXTERNAL_CATALOG_BRIDGE_HANDLER: SyncExternalCatalogBridgeHandler = SyncExternalCatalogBridgeHandler {}; -static GET_TABLE_CATALOG_DIAGNOSTICS_HANDLER: GetTableCatalogDiagnosticsHandler = GetTableCatalogDiagnosticsHandler {}; -static RECOVER_TABLE_CATALOG_HANDLER: RecoverTableCatalogHandler = RecoverTableCatalogHandler {}; -static ROLLBACK_TABLE_CATALOG_HANDLER: RollbackTableCatalogHandler = RollbackTableCatalogHandler {}; - -#[derive(Debug, Serialize)] -struct CatalogConfigResponse { - defaults: BTreeMap, - overrides: BTreeMap, - endpoints: Vec<&'static str>, - admin_discovery: CatalogAdminDiscovery, -} - -#[derive(Debug, Serialize)] -struct CatalogAdminDiscovery { - #[serde(rename = "runtimeCapabilities")] - runtime_capabilities: &'static str, - #[serde(rename = "clusterSnapshot")] - cluster_snapshot: &'static str, - #[serde(rename = "extensionsCatalog")] - extensions_catalog: &'static str, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct CreateNamespaceRequest { - namespace: Vec, - #[serde(default)] - properties: BTreeMap, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RegisterTableRequest { - name: String, - #[serde(rename = "metadata-location")] - metadata_location: String, - #[serde(default)] - overwrite: bool, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct CreateTableRequest { - name: String, - #[serde(default)] - location: Option, - schema: serde_json::Value, - #[serde(default, rename = "partition-spec")] - partition_spec: Option, - #[serde(default, rename = "write-order")] - write_order: Option, - #[serde(default, rename = "stage-create")] - stage_create: bool, - #[serde(default)] - properties: BTreeMap, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct CreateViewRequest { - name: String, - #[serde(default)] - location: Option, - schema: serde_json::Value, - #[serde(rename = "view-version")] - view_version: serde_json::Value, - #[serde(default)] - properties: BTreeMap, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RestCommitTableRequest { - #[serde(default, rename = "identifier")] - _identifier: Option, - #[serde(default, rename = "commit-id")] - commit_id: Option, - #[serde(default, rename = "idempotency-key")] - idempotency_key: Option, - #[serde(default)] - operation: Option, - #[serde(default, rename = "expected-version-token")] - expected_version_token: Option, - #[serde(default, rename = "expected-metadata-location")] - expected_metadata_location: Option, - #[serde(default, rename = "new-metadata-location")] - new_metadata_location: Option, - #[serde(default)] - requirements: Vec, - #[serde(default)] - updates: Vec, - #[serde(default)] - writer: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RestCommitViewRequest { - #[serde(default, rename = "commit-id")] - commit_id: Option, - #[serde(default, rename = "expected-version-token")] - expected_version_token: Option, - #[serde(default, rename = "expected-metadata-location")] - expected_metadata_location: Option, - #[serde(default, rename = "new-metadata-location")] - new_metadata_location: Option, - #[serde(default)] - requirements: Vec, - #[serde(default)] - updates: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct PutTableRefRequest { - #[serde(rename = "snapshot-id")] - snapshot_id: i64, - #[serde(rename = "type")] - ref_type: String, - #[serde(default, rename = "expected-snapshot-id")] - expected_snapshot_id: Option, - #[serde(default, rename = "min-snapshots-to-keep")] - min_snapshots_to_keep: Option, - #[serde(default, rename = "max-snapshot-age-ms")] - max_snapshot_age_ms: Option, - #[serde(default, rename = "max-ref-age-ms")] - max_ref_age_ms: Option, - #[serde(default, rename = "commit-id")] - commit_id: Option, - #[serde(default, rename = "idempotency-key")] - idempotency_key: Option, - #[serde(default)] - writer: Option, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -struct DeleteTableRefRequest { - #[serde(default, rename = "expected-snapshot-id")] - expected_snapshot_id: Option, - #[serde(default)] - force: bool, - #[serde(default, rename = "commit-id")] - commit_id: Option, - #[serde(default, rename = "idempotency-key")] - idempotency_key: Option, - #[serde(default)] - writer: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct TableMetadataMaintenanceRequest { - #[serde(default, rename = "retain-recent-metadata-files")] - retain_recent_metadata_files: usize, - #[serde(default)] - delete: bool, - #[serde(default, rename = "snapshot-expiration")] - snapshot_expiration: Option, - #[serde(default, rename = "commit-snapshot-expiration")] - commit_snapshot_expiration: bool, - #[serde(default)] - compaction: Option, - #[serde(default, rename = "commit-compaction")] - commit_compaction: bool, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -struct TableMaintenanceSchedulerRunRequest { - #[serde(default, rename = "scheduler-id")] - scheduler_id: Option, -} - -impl TableMaintenanceSchedulerRunRequest { - fn scheduler_id(&self) -> &str { - self.scheduler_id.as_deref().unwrap_or(DEFAULT_TABLE_MAINTENANCE_SCHEDULER_ID) - } -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct TableMaintenanceWorkerRunRequest { - #[serde(default, rename = "worker-id")] - worker_id: Option, -} - -impl TableMaintenanceWorkerRunRequest { - fn worker_id(&self) -> &str { - self.worker_id.as_deref().unwrap_or(DEFAULT_TABLE_MAINTENANCE_WORKER_ID) - } -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct TableMaintenanceHeartbeatRequest { - #[serde(rename = "lease-id")] - lease_id: String, - #[serde(rename = "worker-id")] - worker_id: String, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct UpdateTableMetadataLocationRequest { - #[serde(rename = "metadata-location", alias = "metadataLocation")] - metadata_location: String, - #[serde(rename = "version-token", alias = "versionToken")] - version_token: String, - #[serde(default, rename = "commit-id", alias = "commitId")] - commit_id: Option, - #[serde(default, rename = "idempotency-key", alias = "idempotencyKey")] - idempotency_key: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct CatalogImportRequest { - #[serde(rename = "metadata-location", alias = "metadataLocation")] - metadata_location: String, - #[serde(default)] - properties: BTreeMap, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ExternalCatalogBridgeRequest { - catalog: String, - #[serde(default, rename = "external-catalog-id", alias = "externalCatalogId")] - external_catalog_id: Option, - #[serde(rename = "external-namespace", alias = "externalNamespace")] - external_namespace: String, - #[serde(rename = "external-table", alias = "externalTable")] - external_table: String, - #[serde(default, rename = "external-table-uuid", alias = "externalTableUuid")] - external_table_uuid: Option, - #[serde(default, rename = "metadata-location", alias = "metadataLocation")] - metadata_location: Option, - #[serde(default, rename = "external-version-token", alias = "externalVersionToken")] - external_version_token: Option, - #[serde(default, rename = "policy-mode", alias = "policyMode")] - policy_mode: Option, - #[serde(default, rename = "credential-mode", alias = "credentialMode")] - credential_mode: Option, - #[serde(default, rename = "sync-mode", alias = "syncMode")] - sync_mode: Option, - #[serde(default)] - properties: BTreeMap, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ExternalCatalogBridgeSyncRequest { - catalog: String, - #[serde(default, rename = "external-catalog-id", alias = "externalCatalogId")] - external_catalog_id: Option, - #[serde(rename = "external-namespace", alias = "externalNamespace")] - external_namespace: String, - #[serde(rename = "external-table", alias = "externalTable")] - external_table: String, - #[serde(default, rename = "external-table-uuid", alias = "externalTableUuid")] - external_table_uuid: Option, - #[serde(rename = "metadata-location", alias = "metadataLocation")] - metadata_location: String, - #[serde(default, rename = "external-version-token", alias = "externalVersionToken")] - external_version_token: Option, - #[serde(default, rename = "expected-version-token", alias = "expectedVersionToken")] - expected_version_token: Option, - #[serde(default, rename = "expected-metadata-location", alias = "expectedMetadataLocation")] - expected_metadata_location: Option, - #[serde(default, rename = "commit-id", alias = "commitId")] - commit_id: Option, - #[serde(default, rename = "idempotency-key", alias = "idempotencyKey")] - idempotency_key: Option, - #[serde(default, rename = "policy-mode", alias = "policyMode")] - policy_mode: Option, - #[serde(default, rename = "credential-mode", alias = "credentialMode")] - credential_mode: Option, - #[serde(default, rename = "rollback-strategy", alias = "rollbackStrategy")] - rollback_strategy: Option, - #[serde(default)] - properties: BTreeMap, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RollbackTableRequest { - #[serde(rename = "metadata-location", alias = "metadataLocation")] - metadata_location: String, - #[serde(rename = "version-token", alias = "versionToken")] - version_token: String, - #[serde(default, rename = "commit-id", alias = "commitId")] - commit_id: Option, - #[serde(default, rename = "idempotency-key", alias = "idempotencyKey")] - idempotency_key: Option, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "kebab-case")] -struct TableRefsResponse { - table_bucket: String, - namespace: String, - table: String, - current_metadata_location: String, - current_snapshot_id: Option, - protected_ref_count: usize, - user_defined_ref_count: usize, - refs: BTreeMap, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "kebab-case")] -struct ExternalCatalogBridgeResponse { - table_bucket: String, - namespace: String, - table: String, - status: String, - supported_import: String, - #[serde(default)] - capabilities: Vec, - #[serde(default)] - unsupported_bridges: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - bridge: Option, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "kebab-case")] -struct ExternalCatalogBridgeCapability { - catalog: String, - status: String, - reason: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "kebab-case")] -struct ExternalCatalogBridgeStateResponse { - catalog: String, - external_catalog_id: Option, - external_namespace: String, - external_table: String, - external_table_uuid: Option, - metadata_location: Option, - external_version_token: Option, - policy_mode: String, - credential_mode: String, - sync_mode: String, - rollback_strategy: String, - last_sync_status: Option, - last_synced_metadata_location: Option, - properties: BTreeMap, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "kebab-case")] -struct ExternalCatalogBridgeSyncResponse { - action: String, - table: RestLoadTableResponse, - bridge: ExternalCatalogBridgeResponse, -} - -#[derive(Debug, Serialize)] -struct TableBucketResponse { - #[serde(rename = "table-bucket")] - table_bucket: String, - enabled: bool, - #[serde(rename = "catalog-type")] - catalog_type: String, - warehouse: String, - #[serde(rename = "warehouse-location")] - warehouse_location: String, - #[serde(rename = "catalog-uri")] - catalog_uri: String, - #[serde(rename = "compat-catalog-uri")] - compat_catalog_uri: String, - #[serde(rename = "credential-vending")] - credential_vending: &'static str, - #[serde(rename = "credential-scope")] - credential_scope: &'static str, - #[serde(rename = "credential-scope-prefix")] - credential_scope_prefix: String, - #[serde(rename = "catalog-entry-present")] - catalog_entry_present: bool, - properties: BTreeMap, -} - -#[derive(Debug, Serialize)] -struct RestNamespaceResponse { - namespace: Vec, - properties: BTreeMap, -} - -#[derive(Debug, Serialize)] -struct RestListNamespacesResponse { - namespaces: Vec>, - #[serde(rename = "next-page-token")] - next_page_token: Option, -} - -#[derive(Debug, Serialize)] -struct RestTableIdentifier { - namespace: Vec, - name: String, -} - -#[derive(Debug, Serialize)] -struct RestListTablesResponse { - identifiers: Vec, - #[serde(rename = "next-page-token")] - next_page_token: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct RestPageToken { - version: u8, - context: String, - cursor: String, -} - -#[derive(Debug)] -enum RestPagination { - Unpaginated, - Paginated { - cursor: Option, - limit: NonZeroUsize, - context: String, - }, -} - -impl RestPagination { - fn page_request(&self) -> Option<(Option<&str>, NonZeroUsize)> { - match self { - Self::Unpaginated => None, - Self::Paginated { cursor, limit, .. } => Some((cursor.as_deref(), *limit)), - } - } - - fn next_page_token(&self, cursor: Option) -> S3Result> { - match (self, cursor) { - (Self::Unpaginated, _) | (_, None) => Ok(None), - (Self::Paginated { context, .. }, Some(cursor)) => encode_rest_page_token(&cursor, context).map(Some), - } - } -} - -#[derive(Clone, Copy)] -struct RestPageContext<'a> { - resource: &'static str, - warehouse: &'a str, - namespace: Option<&'a str>, -} - -#[derive(Debug, Serialize)] -struct RestListViewsResponse { - identifiers: Vec, - #[serde(rename = "next-page-token")] - next_page_token: Option, -} - -#[derive(Debug, Serialize)] -struct RestLoadViewResponse { - #[serde(rename = "metadata-location")] - metadata_location: String, - metadata: serde_json::Value, - config: BTreeMap, -} - -#[derive(Debug, Serialize)] -struct RestStorageCredential { - prefix: String, - config: BTreeMap, -} - -#[derive(Debug, Clone)] -struct TableCredentialScope { - scope_prefix: String, - object_prefix: String, -} - -#[derive(Debug, Clone)] -struct TableCredentialIssueRequest<'a> { - entry: &'a crate::table_catalog::TableEntry, - principal: Option<&'a rustfs_credentials::Credentials>, - scope_prefix: String, - object_prefix: String, -} - -#[derive(Debug, Clone)] -struct IssuedTableCredentials { - access_key_id: String, - secret_access_key: String, - session_token: String, - expiration: OffsetDateTime, -} - -#[async_trait::async_trait] -trait TableCredentialIssuer: Sync { - fn enabled(&self) -> bool { - true - } - - async fn issue_table_credentials(&self, request: TableCredentialIssueRequest<'_>) - -> S3Result>; -} - -#[cfg(test)] -struct DisabledTableCredentialIssuer; - -#[cfg(test)] -#[async_trait::async_trait] -impl TableCredentialIssuer for DisabledTableCredentialIssuer { - fn enabled(&self) -> bool { - false - } - - async fn issue_table_credentials( - &self, - _request: TableCredentialIssueRequest<'_>, - ) -> S3Result> { - Ok(None) - } -} - -struct IamTableCredentialIssuer { - enabled: bool, - ttl_seconds: i64, -} - -impl IamTableCredentialIssuer { - fn from_env() -> Self { - Self { - enabled: table_credential_vending_enabled(), - ttl_seconds: table_credential_ttl_seconds(), - } - } -} - -#[async_trait::async_trait] -impl TableCredentialIssuer for IamTableCredentialIssuer { - fn enabled(&self) -> bool { - self.enabled - } - - async fn issue_table_credentials( - &self, - request: TableCredentialIssueRequest<'_>, - ) -> S3Result> { - if !self.enabled { - return Ok(None); - } - - let Some(principal) = request.principal else { - return Err(s3_error!(InvalidRequest, "authentication required for table credentials")); - }; - if principal.is_temp() || principal.is_service_account() { - return Err(s3_error!( - AccessDenied, - "table credential vending does not allow chained temporary credentials" - )); - } - - let policy = table_credential_session_policy(request.entry, &request.object_prefix)?; - let policy_buf = serde_json::to_vec(&policy) - .map_err(|err| s3_error!(InternalError, "failed to serialize table credential session policy: {}", err))?; - let expiration = OffsetDateTime::now_utc().saturating_add(Duration::seconds(self.ttl_seconds)); - let mut claims: HashMap = principal.claims.clone().unwrap_or_default(); - claims.insert( - "exp".to_string(), - serde_json::Value::Number(serde_json::Number::from(expiration.unix_timestamp())), - ); - claims.insert("parent".to_string(), serde_json::Value::String(principal.access_key.clone())); - claims.insert( - SESSION_POLICY_NAME.to_string(), - serde_json::Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&policy_buf)), - ); - claims.insert( - "rustfs:table-bucket".to_string(), - serde_json::Value::String(request.entry.table_bucket.clone()), - ); - claims.insert("rustfs:table-id".to_string(), serde_json::Value::String(request.entry.table_id.clone())); - claims.insert( - "rustfs:credential-scope-prefix".to_string(), - serde_json::Value::String(request.scope_prefix.clone()), - ); - - let secret = current_token_signing_key().ok_or_else(|| s3_error!(InternalError, "token signing key not initialized"))?; - let mut credential = get_new_credentials_with_metadata(&claims, &secret) - .map_err(|err| s3_error!(InternalError, "failed to generate table credentials: {}", err))?; - bind_table_credential_parent(&mut credential, principal); - - let iam_store = - crate::admin::runtime_sources::current_ready_iam_handle().map_err(|_| s3_error!(InternalError, "iam not init"))?; - iam_store - .set_temp_user(&credential.access_key, &credential, None) - .await - .map_err(|_| s3_error!(InternalError, "failed to store table credentials"))?; - - Ok(Some(IssuedTableCredentials { - access_key_id: credential.access_key, - secret_access_key: credential.secret_key, - session_token: credential.session_token, - expiration, - })) - } -} - -fn bind_table_credential_parent(credential: &mut rustfs_credentials::Credentials, principal: &rustfs_credentials::Credentials) { - credential.parent_user = principal.access_key.clone(); -} - -#[derive(Debug, Serialize)] -struct RestLoadTableResponse { - #[serde(rename = "metadata-location")] - metadata_location: String, - metadata: serde_json::Value, - config: BTreeMap, - #[serde(rename = "storage-credentials")] - storage_credentials: Vec, -} - -#[derive(Debug, Serialize)] -struct RestLoadCredentialsResponse { - config: BTreeMap, - #[serde(rename = "storage-credentials")] - storage_credentials: Vec, -} - -#[derive(Debug, Serialize)] -struct RestCommitTableResponse { - #[serde(rename = "metadata-location")] - metadata_location: String, - metadata: serde_json::Value, - #[serde(rename = "version-token")] - version_token: String, - generation: u64, - #[serde(rename = "commit-id")] - commit_id: String, -} - -#[derive(Debug, Serialize)] -struct TableMetadataLocationResponse { - #[serde(rename = "metadata-location")] - metadata_location: String, - #[serde(rename = "version-token")] - version_token: String, - generation: u64, - #[serde(rename = "warehouse-location")] - warehouse_location: String, -} - -pub fn register_table_catalog_route(r: &mut S3Router) -> std::io::Result<()> { - for prefix in [TABLE_CATALOG_PREFIX, TABLE_CATALOG_COMPAT_PREFIX] { - register_table_catalog_prefix_routes(r, prefix)?; - } - - Ok(()) -} - -fn register_table_catalog_prefix_routes(r: &mut S3Router, prefix: &str) -> std::io::Result<()> { - r.insert(Method::GET, format!("{prefix}/config").as_str(), AdminOperation(&GET_CONFIG_HANDLER))?; - r.insert( - Method::PUT, - format!("{prefix}/buckets/{{warehouse}}").as_str(), - AdminOperation(&ENABLE_TABLE_BUCKET_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/buckets/{{warehouse}}").as_str(), - AdminOperation(&GET_TABLE_BUCKET_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/catalog/migration").as_str(), - AdminOperation(&GET_TABLE_CATALOG_MIGRATION_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/catalog/migration").as_str(), - AdminOperation(&MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER), - )?; - r.insert( - Method::DELETE, - format!("{prefix}/{{warehouse}}/catalog/migration").as_str(), - AdminOperation(&CANCEL_TABLE_CATALOG_MIGRATION_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces").as_str(), - AdminOperation(&LIST_NAMESPACES_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces").as_str(), - AdminOperation(&CREATE_NAMESPACE_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}").as_str(), - AdminOperation(&GET_NAMESPACE_HANDLER), - )?; - r.insert( - Method::HEAD, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}").as_str(), - AdminOperation(&NAMESPACE_EXISTS_HANDLER), - )?; - r.insert( - Method::DELETE, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}").as_str(), - AdminOperation(&DROP_NAMESPACE_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables").as_str(), - AdminOperation(&LIST_TABLES_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables").as_str(), - AdminOperation(&CREATE_TABLE_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/register").as_str(), - AdminOperation(®ISTER_TABLE_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views").as_str(), - AdminOperation(&LIST_VIEWS_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views").as_str(), - AdminOperation(&CREATE_VIEW_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}").as_str(), - AdminOperation(&LOAD_TABLE_HANDLER), - )?; - r.insert( - Method::HEAD, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}").as_str(), - AdminOperation(&TABLE_EXISTS_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/credentials").as_str(), - AdminOperation(&LOAD_CREDENTIALS_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}").as_str(), - AdminOperation(&COMMIT_TABLE_HANDLER), - )?; - r.insert( - Method::DELETE, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}").as_str(), - AdminOperation(&DROP_TABLE_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views/{{view}}").as_str(), - AdminOperation(&LOAD_VIEW_HANDLER), - )?; - r.insert( - Method::HEAD, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views/{{view}}").as_str(), - AdminOperation(&VIEW_EXISTS_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views/{{view}}").as_str(), - AdminOperation(&REPLACE_VIEW_HANDLER), - )?; - r.insert( - Method::DELETE, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views/{{view}}").as_str(), - AdminOperation(&DROP_VIEW_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/refs").as_str(), - AdminOperation(&LIST_TABLE_REFS_HANDLER), - )?; - r.insert( - Method::PUT, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/refs/{{ref}}").as_str(), - AdminOperation(&PUT_TABLE_REF_HANDLER), - )?; - r.insert( - Method::DELETE, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/refs/{{ref}}").as_str(), - AdminOperation(&DELETE_TABLE_REF_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/metadata-location").as_str(), - AdminOperation(&GET_TABLE_METADATA_LOCATION_HANDLER), - )?; - r.insert( - Method::PUT, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/metadata-location").as_str(), - AdminOperation(&UPDATE_TABLE_METADATA_LOCATION_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/metadata").as_str(), - AdminOperation(&TABLE_METADATA_MAINTENANCE_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/config").as_str(), - AdminOperation(&GET_TABLE_MAINTENANCE_CONFIG_HANDLER), - )?; - r.insert( - Method::PUT, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/config").as_str(), - AdminOperation(&PUT_TABLE_MAINTENANCE_CONFIG_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/jobs/{{job}}").as_str(), - AdminOperation(&GET_TABLE_MAINTENANCE_JOB_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/scheduler").as_str(), - AdminOperation(&GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/scheduler/run").as_str(), - AdminOperation(&RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/worker/run").as_str(), - AdminOperation(&RUN_TABLE_MAINTENANCE_WORKER_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/jobs/{{job}}/heartbeat").as_str(), - AdminOperation(&HEARTBEAT_TABLE_MAINTENANCE_JOB_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/jobs/{{job}}/quarantine").as_str(), - AdminOperation(&TABLE_MAINTENANCE_QUARANTINE_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/export").as_str(), - AdminOperation(&EXPORT_TABLE_CATALOG_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/import").as_str(), - AdminOperation(&IMPORT_TABLE_CATALOG_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/external").as_str(), - AdminOperation(&EXTERNAL_CATALOG_BRIDGE_HANDLER), - )?; - r.insert( - Method::PUT, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/external").as_str(), - AdminOperation(&PUT_EXTERNAL_CATALOG_BRIDGE_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/external/sync").as_str(), - AdminOperation(&SYNC_EXTERNAL_CATALOG_BRIDGE_HANDLER), - )?; - r.insert( - Method::GET, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/diagnostics").as_str(), - AdminOperation(&GET_TABLE_CATALOG_DIAGNOSTICS_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/recovery").as_str(), - AdminOperation(&RECOVER_TABLE_CATALOG_HANDLER), - )?; - r.insert( - Method::POST, - format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/rollback").as_str(), - AdminOperation(&ROLLBACK_TABLE_CATALOG_HANDLER), - )?; - - Ok(()) -} - -fn catalog_config_response(warehouse: Option<&str>) -> S3Result { - let usecase = default_admin_usecase(); - let backing_mode = crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)?; - let mut overrides = BTreeMap::new(); - if backing_mode != crate::table_catalog::TableCatalogBackingMode::ObjectBacked { - overrides.insert(CATALOG_BACKING_CONFIG_KEY.to_string(), backing_mode.as_str().to_string()); - } - let mut defaults = BTreeMap::from([ - (WAREHOUSE_PROPERTY.to_string(), DEFAULT_WAREHOUSE_ID.to_string()), - (CATALOG_ENDPOINT_PREFIX_CONFIG_KEY.to_string(), TABLE_CATALOG_PREFIX.to_string()), - ( - CATALOG_COMPAT_ENDPOINT_PREFIX_CONFIG_KEY.to_string(), - TABLE_CATALOG_COMPAT_PREFIX.to_string(), - ), - ( - CATALOG_BACKING_CONFIG_KEY.to_string(), - crate::table_catalog::TABLE_CATALOG_BACKING_OBJECT.to_string(), - ), - ]); - if let Some(warehouse) = warehouse { - defaults.insert(PREFIX_PROPERTY.to_string(), warehouse.to_string()); - } - Ok(CatalogConfigResponse { - defaults, - overrides, - endpoints: TABLE_CATALOG_ENDPOINTS.to_vec(), - admin_discovery: CatalogAdminDiscovery { - runtime_capabilities: usecase.runtime_capabilities_route(), - cluster_snapshot: usecase.cluster_snapshot_route(), - extensions_catalog: usecase.extensions_catalog_route(), - }, - }) -} - -fn build_json_response(status: StatusCode, body: &T) -> S3Result> { - let data = serde_json::to_vec(body).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; - let mut headers = HeaderMap::new(); - headers.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE)); - Ok(S3Response::with_headers((status, Body::from(data)), headers)) -} - -fn empty_response(status: StatusCode) -> S3Response<(StatusCode, Body)> { - S3Response::new((status, Body::default())) -} - -fn duration_millis_u64(duration: StdDuration) -> u64 { - u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) -} - -fn table_catalog_admin_operation_result_label(result: &Result) -> &'static str { - if result.is_ok() { "success" } else { "failure" } -} - -fn record_table_catalog_admin_operation_result( - operation: &str, - warehouse: &str, - namespace: &str, - table: &str, - started: Instant, - result: &Result, -) { - let elapsed = started.elapsed(); - let result_label = table_catalog_admin_operation_result_label(result); - counter!( - "rustfs_table_catalog_admin_operations_total", - "operation" => operation.to_string(), - "result" => result_label.to_string() - ) - .increment(1); - histogram!( - "rustfs_table_catalog_admin_operation_duration_seconds", - "operation" => operation.to_string(), - "result" => result_label.to_string() - ) - .record(elapsed.as_secs_f64()); - - if result.is_err() { - tracing::warn!( - operation, - warehouse, - namespace, - table, - result = result_label, - duration_ms = duration_millis_u64(elapsed), - "table catalog admin operation failed" - ); - } else if elapsed >= TABLE_CATALOG_ADMIN_OPERATION_SLOW_LOG_THRESHOLD { - tracing::warn!( - operation, - warehouse, - namespace, - table, - duration_ms = duration_millis_u64(elapsed), - "slow table catalog admin operation" - ); - } -} - -fn exists_status(exists: bool) -> StatusCode { - if exists { - StatusCode::NO_CONTENT - } else { - StatusCode::NOT_FOUND - } -} - -async fn authorize_table_catalog_request(req: &S3Request, action: AdminAction) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(action)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await -} - -#[derive(Debug, Clone)] -struct TableCatalogResource<'a> { - warehouse: &'a str, - namespace: Option, - table: Option, - view: Option, -} - -impl<'a> TableCatalogResource<'a> { - fn warehouse(warehouse: &'a str) -> Self { - Self { - warehouse, - namespace: None, - table: None, - view: None, - } - } - - fn namespace(warehouse: &'a str, namespace: &crate::table_catalog::Namespace) -> Self { - Self { - warehouse, - namespace: Some(namespace.storage_id()), - table: None, - view: None, - } - } - - fn table(warehouse: &'a str, namespace: &crate::table_catalog::Namespace, table: &str) -> Self { - Self { - warehouse, - namespace: Some(namespace.storage_id()), - table: Some(table.to_string()), - view: None, - } - } - - fn view(warehouse: &'a str, namespace: &crate::table_catalog::Namespace, view: &str) -> Self { - Self { - warehouse, - namespace: Some(namespace.storage_id()), - table: None, - view: Some(view.to_string()), - } - } - - fn object_path(&self) -> Option { - match (&self.namespace, &self.table, &self.view) { - (Some(namespace), Some(table), None) => Some(format!( - "{TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT}/{namespace}/{TABLE_CATALOG_TABLE_RESOURCE_ROOT}/{table}" - )), - (Some(namespace), None, Some(view)) => Some(format!( - "{TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT}/{namespace}/{TABLE_CATALOG_VIEW_RESOURCE_ROOT}/{view}" - )), - (Some(namespace), None, None) => Some(format!("{TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT}/{namespace}")), - _ => None, - } - } -} - -async fn authorize_table_catalog_resource_request( - req: &S3Request, - resource: &TableCatalogResource<'_>, - action: AdminAction, -) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - let object_path = resource.object_path(); - validate_admin_request_with_bucket_object( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(action)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - AdminResourceScope::bucket_object(resource.warehouse, object_path.as_deref().unwrap_or("")), - ) - .await -} - -async fn table_catalog_request_principal(req: &S3Request) -> S3Result { - let Some(input_cred) = &req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - let (cred, _owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - Ok(cred) -} - -async fn read_json_body(mut input: Body) -> S3Result { - let body = input - .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) - .await - .map_err(|err| s3_error!(InvalidRequest, "failed to read request body: {}", err))?; - if body.is_empty() { - return Err(s3_error!(InvalidRequest, "request body is required")); - } - serde_json::from_slice(&body).map_err(|err| s3_error!(InvalidRequest, "invalid JSON: {}", err)) -} - -async fn read_json_body_or_default(mut input: Body) -> S3Result -where - T: Default + DeserializeOwned, -{ - let body = input - .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) - .await - .map_err(|err| s3_error!(InvalidRequest, "failed to read request body: {}", err))?; - if body.is_empty() { - return Ok(T::default()); - } - serde_json::from_slice(&body).map_err(|err| s3_error!(InvalidRequest, "invalid JSON: {}", err)) -} - -fn warehouse_from_params(params: &Params<'_, '_>) -> S3Result { - let warehouse = params.get("warehouse").unwrap_or(""); - if warehouse.is_empty() { - return Err(s3_error!(InvalidRequest, "warehouse is required")); - } - Ok(warehouse.to_string()) -} - -fn warehouse_from_config_query(uri: &http::Uri) -> S3Result> { - let Some(query) = uri.query() else { - return Ok(None); - }; - let mut warehouse = None; - for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { - if key != WAREHOUSE_PROPERTY { - continue; - } - if warehouse.is_some() { - return Err(s3_error!(InvalidRequest, "warehouse query parameter must not be repeated")); - } - if value.is_empty() { - return Err(s3_error!(InvalidRequest, "warehouse query parameter must not be empty")); - } - warehouse = Some(value.into_owned()); - } - Ok(warehouse) -} - -fn rest_pagination_from_query(uri: &http::Uri, context: RestPageContext<'_>) -> S3Result { - let mut page_token = None; - let mut page_token_seen = false; - let mut page_size = None; - - if let Some(query) = uri.query() { - for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - REST_PAGE_TOKEN_QUERY_PARAMETER => { - if page_token_seen { - return Err(iceberg_rest_error( - ICEBERG_ERROR_BAD_REQUEST, - StatusCode::BAD_REQUEST, - "pageToken query parameter must not be repeated", - )); - } - page_token_seen = true; - page_token = Some(value.into_owned()); - } - REST_PAGE_SIZE_QUERY_PARAMETER => { - if page_size.is_some() { - return Err(iceberg_rest_error( - ICEBERG_ERROR_BAD_REQUEST, - StatusCode::BAD_REQUEST, - "pageSize query parameter must not be repeated", - )); - } - let value = value.parse::().map_err(|_| { - iceberg_rest_error( - ICEBERG_ERROR_BAD_REQUEST, - StatusCode::BAD_REQUEST, - "pageSize query parameter must be a positive integer", - ) - })?; - if value == 0 { - return Err(iceberg_rest_error( - ICEBERG_ERROR_BAD_REQUEST, - StatusCode::BAD_REQUEST, - "pageSize query parameter must be greater than zero", - )); - } - page_size = Some(value.min(REST_MAX_PAGE_SIZE)); - } - _ => {} - } - } - } - - if !page_token_seen && page_size.is_none() { - return Ok(RestPagination::Unpaginated); - } - - let context = rest_page_context_fingerprint(context); - let cursor = match page_token.as_deref() { - None | Some("") => None, - Some(encoded) => Some(decode_rest_page_token(encoded, &context)?), - }; - let limit = NonZeroUsize::new(page_size.unwrap_or(REST_DEFAULT_PAGE_SIZE)).ok_or_else(|| { - iceberg_rest_error( - ICEBERG_ERROR_REST, - StatusCode::INTERNAL_SERVER_ERROR, - "REST page size must be greater than zero", - ) - })?; - Ok(RestPagination::Paginated { cursor, limit, context }) -} - -fn rest_page_context_fingerprint(context: RestPageContext<'_>) -> String { - let mut data = - Vec::with_capacity(context.resource.len() + context.warehouse.len() + context.namespace.map_or(0, str::len) + 2); - data.extend_from_slice(context.resource.as_bytes()); - data.push(0); - data.extend_from_slice(context.warehouse.as_bytes()); - data.push(0); - if let Some(namespace) = context.namespace { - data.extend_from_slice(namespace.as_bytes()); - } - hex_sha256(&data, str::to_string) -} - -fn decode_rest_page_token(encoded: &str, expected_context: &str) -> S3Result { - if encoded.len() > REST_PAGE_TOKEN_MAX_LENGTH { - return Err(iceberg_rest_error( - ICEBERG_ERROR_BAD_REQUEST, - StatusCode::BAD_REQUEST, - "pageToken query parameter is too large", - )); - } - let data = base64_decode_url_safe_no_pad(encoded.as_bytes()).map_err(|_| { - iceberg_rest_error( - ICEBERG_ERROR_BAD_REQUEST, - StatusCode::BAD_REQUEST, - "pageToken query parameter is malformed", - ) - })?; - let token = serde_json::from_slice::(&data).map_err(|_| { - iceberg_rest_error( - ICEBERG_ERROR_BAD_REQUEST, - StatusCode::BAD_REQUEST, - "pageToken query parameter is malformed", - ) - })?; - if token.version != REST_PAGE_TOKEN_VERSION || token.context != expected_context || token.cursor.is_empty() { - return Err(iceberg_rest_error( - ICEBERG_ERROR_BAD_REQUEST, - StatusCode::BAD_REQUEST, - "pageToken query parameter does not match this list operation", - )); - } - Ok(token.cursor) -} - -fn encode_rest_page_token(cursor: &str, context: &str) -> S3Result { - let token = RestPageToken { - version: REST_PAGE_TOKEN_VERSION, - context: context.to_string(), - cursor: cursor.to_string(), - }; - let data = serde_json::to_vec(&token).map_err(|err| { - iceberg_rest_error( - ICEBERG_ERROR_REST, - StatusCode::INTERNAL_SERVER_ERROR, - format!("failed to serialize REST page token: {err}"), - ) - })?; - let encoded = base64_encode_url_safe_no_pad(&data); - if encoded.len() > REST_PAGE_TOKEN_MAX_LENGTH { - return Err(iceberg_rest_error( - ICEBERG_ERROR_REST, - StatusCode::INTERNAL_SERVER_ERROR, - "REST page token exceeds the supported size", - )); - } - Ok(encoded) -} - -fn namespace_from_params(params: &Params<'_, '_>) -> S3Result { - let namespace = params.get("namespace").unwrap_or(""); - crate::table_catalog::Namespace::parse(namespace).map_err(|err| s3_error!(InvalidRequest, "invalid namespace: {}", err)) -} - -fn table_name_from_params(params: &Params<'_, '_>) -> S3Result { - let table = params.get("table").unwrap_or(""); - crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - Ok(table.to_string()) -} - -fn view_name_from_params(params: &Params<'_, '_>) -> S3Result { - let view = params.get("view").unwrap_or(""); - crate::table_catalog::IdentifierSegment::parse(view.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid view name: {}", err))?; - Ok(view.to_string()) -} - -fn ref_name_from_params(params: &Params<'_, '_>) -> S3Result { - let ref_name = params.get("ref").unwrap_or(""); - crate::table_catalog::IdentifierSegment::parse(ref_name.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid ref name: {}", err))?; - Ok(ref_name.to_string()) -} - -fn job_id_from_params(params: &Params<'_, '_>) -> S3Result { - let job = params.get("job").unwrap_or(""); - if job.is_empty() { - return Err(s3_error!(InvalidRequest, "maintenance job id is required")); - } - Ok(job.to_string()) -} - -fn table_catalog_backend() -> S3Result> { - let store = current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store not initialized"))?; - Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new(store)) -} - -type EcStoreObjectTableCatalogStore = - crate::table_catalog::ObjectTableCatalogStore>; - -fn table_catalog_store_from_backend( - backend: crate::table_catalog::EcStoreTableCatalogObjectBackend, -) -> S3Result> { - crate::table_catalog::ConfiguredTableCatalogStore::from_env(backend).map_err(catalog_store_error) -} - -fn table_catalog_store() -> S3Result> { - let backend = table_catalog_backend()?; - table_catalog_store_from_backend(backend) -} - -fn table_catalog_object_store() -> S3Result { - match crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)? { - crate::table_catalog::TableCatalogBackingMode::ObjectBacked => { - let backend = table_catalog_backend()?; - Ok(crate::table_catalog::ObjectTableCatalogStore::new(backend)) - } - crate::table_catalog::TableCatalogBackingMode::DurableStrong => Err(s3_error!( - InvalidRequest, - "operation is not supported with {} table catalog backing", - crate::table_catalog::TABLE_CATALOG_BACKING_DURABLE_STRONG - )), - } -} - -async fn table_bucket_enabled_from_metadata(bucket: &str) -> S3Result { - let metadata = metadata_sys::get(bucket) - .await - .map_err(|err| s3_error!(InvalidRequest, "failed to load table bucket metadata for {bucket}: {}", err))?; - Ok(metadata.table_bucket_enabled()) -} - -async fn ensure_table_bucket_enabled(bucket: &str) -> S3Result<()> { - if table_bucket_enabled_from_metadata(bucket).await? { - return Ok(()); - } - Err(s3_error!(InvalidRequest, "bucket {bucket} is not table-enabled")) -} - -fn table_bucket_entry_from_metadata_marker(bucket: &str) -> crate::table_catalog::TableBucketEntry { - crate::table_catalog::TableBucketEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - catalog_type: crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(), - warehouse_root: format!("s3://{bucket}/"), - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - } -} - -async fn enable_table_bucket_marker(bucket: &str) -> S3Result<()> { - let marker = crate::table_catalog::table_bucket_marker_json() - .map_err(|err| s3_error!(InternalError, "failed to serialize table bucket marker: {}", err))?; - metadata_sys::update(bucket, crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG, marker) - .await - .map(|_| ()) - .map_err(|err| s3_error!(InvalidRequest, "failed to enable table bucket {bucket}: {}", err)) -} - -async fn ensure_table_bucket_entry(store: &S, bucket: &str, table_bucket_enabled: bool) -> S3Result<()> -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - if !table_bucket_enabled { - return Err(s3_error!(InvalidRequest, "bucket {bucket} is not table-enabled")); - } - if store.get_table_bucket(bucket).await.map_err(catalog_store_error)?.is_some() { - return Ok(()); - } - store - .put_table_bucket(table_bucket_entry_from_metadata_marker(bucket)) - .await - .map_err(catalog_store_error) -} - -async fn table_bucket_response(store: &S, bucket: &str, enabled: bool) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let entry = store.get_table_bucket(bucket).await.map_err(catalog_store_error)?; - let (catalog_type, warehouse_location, properties, catalog_entry_present) = match entry { - Some(entry) => (entry.catalog_type, entry.warehouse_root, entry.properties, true), - None => ( - crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(), - format!("s3://{bucket}/"), - BTreeMap::new(), - false, - ), - }; - - Ok(TableBucketResponse { - table_bucket: bucket.to_string(), - enabled, - catalog_type, - warehouse: bucket.to_string(), - warehouse_location: warehouse_location.clone(), - catalog_uri: format!("{TABLE_CATALOG_PREFIX}/{bucket}"), - compat_catalog_uri: format!("{TABLE_CATALOG_COMPAT_PREFIX}/{bucket}"), - credential_vending: CREDENTIAL_VENDING_UNSUPPORTED, - credential_scope: CREDENTIAL_SCOPE_WAREHOUSE_PREFIX, - credential_scope_prefix: warehouse_location, - catalog_entry_present, - properties, - }) -} - -async fn enable_table_bucket_response(store: &S, bucket: &str) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - enable_table_bucket_marker(bucket).await?; - ensure_table_bucket_entry(store, bucket, true).await?; - table_bucket_response(store, bucket, true).await -} - -fn namespace_segments(namespace: &crate::table_catalog::Namespace) -> Vec { - namespace - .segments() - .iter() - .map(|segment| segment.as_str().to_string()) - .collect() -} - -fn namespace_from_segments(segments: &[String]) -> S3Result { - if segments.is_empty() { - return Err(s3_error!(InvalidRequest, "namespace cannot be empty")); - } - - let namespace = segments.join("."); - crate::table_catalog::Namespace::parse(&namespace).map_err(|err| s3_error!(InvalidRequest, "invalid namespace: {}", err)) -} - -fn namespace_response_from_entry(entry: crate::table_catalog::NamespaceEntry) -> S3Result { - let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) - .map_err(|err| s3_error!(InternalError, "persisted namespace entry is invalid: {}", err))?; - Ok(RestNamespaceResponse { - namespace: namespace_segments(&namespace), - properties: entry.properties, - }) -} - -fn list_namespaces_response_from_entries( - entries: Vec, - next_page_token: Option, -) -> S3Result { - let namespaces = entries - .into_iter() - .map(|entry| { - let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) - .map_err(|err| s3_error!(InternalError, "persisted namespace entry is invalid: {}", err))?; - Ok(namespace_segments(&namespace)) - }) - .collect::>>()?; - Ok(RestListNamespacesResponse { - namespaces, - next_page_token, - }) -} - -fn list_tables_response_from_entries( - entries: Vec, - next_page_token: Option, -) -> S3Result { - let identifiers = entries - .into_iter() - .map(|entry| { - let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) - .map_err(|err| s3_error!(InternalError, "persisted table entry namespace is invalid: {}", err))?; - Ok(RestTableIdentifier { - namespace: namespace_segments(&namespace), - name: entry.table, - }) - }) - .collect::>>()?; - Ok(RestListTablesResponse { - identifiers, - next_page_token, - }) -} - -fn list_views_response_from_entries( - entries: Vec, - next_page_token: Option, -) -> S3Result { - let identifiers = entries - .into_iter() - .map(|entry| { - let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) - .map_err(|err| s3_error!(InternalError, "persisted view entry namespace is invalid: {}", err))?; - Ok(RestTableIdentifier { - namespace: namespace_segments(&namespace), - name: entry.view, - }) - }) - .collect::>>()?; - Ok(RestListViewsResponse { - identifiers, - next_page_token, - }) -} - -fn table_credential_vending_enabled() -> bool { - std::env::var(ENV_TABLE_CATALOG_CREDENTIAL_VENDING) - .ok() - .map(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true" | "on" | "enabled")) - .unwrap_or(false) -} - -fn table_credential_ttl_seconds() -> i64 { - std::env::var(ENV_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS) - .ok() - .and_then(|value| value.parse::().ok()) - .map(|seconds| seconds.clamp(MIN_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS, MAX_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS)) - .unwrap_or(DEFAULT_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS) -} - -fn table_credential_scope(entry: &crate::table_catalog::TableEntry) -> S3Result { - let location = entry - .warehouse_location - .strip_prefix("s3://") - .ok_or_else(|| s3_error!(InvalidRequest, "table warehouse location must be an s3 URI"))?; - let (bucket, object_prefix) = location - .split_once('/') - .ok_or_else(|| s3_error!(InvalidRequest, "table warehouse location must include an object prefix"))?; - if bucket != entry.table_bucket { - return Err(s3_error!(InvalidRequest, "table warehouse location must be inside the table bucket")); - } - let object_prefix = normalize_table_credential_object_prefix(object_prefix)?; - Ok(TableCredentialScope { - scope_prefix: format!("s3://{bucket}/{object_prefix}"), - object_prefix, - }) -} - -fn normalize_table_credential_object_prefix(object_prefix: &str) -> S3Result { - let object_prefix = object_prefix.strip_suffix('/').unwrap_or(object_prefix); - if object_prefix.is_empty() { - return Err(s3_error!(InvalidRequest, "table credential scope prefix is empty")); - } - if object_prefix.contains('\\') { - return Err(s3_error!( - InvalidRequest, - "table credential scope prefix contains an invalid path separator" - )); - } - if object_prefix - .split('/') - .any(|segment| segment.is_empty() || segment == "." || segment == "..") - { - return Err(s3_error!( - InvalidRequest, - "table credential scope prefix contains an invalid path segment" - )); - } - - let mut normalized = object_prefix.to_string(); - normalized.push('/'); - Ok(normalized) -} - -fn table_credential_catalog_resource(entry: &crate::table_catalog::TableEntry) -> S3Result { - let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) - .map_err(|err| s3_error!(InvalidRequest, "invalid table credential namespace: {}", err))?; - let table = crate::table_catalog::IdentifierSegment::parse(&entry.table) - .map_err(|err| s3_error!(InvalidRequest, "invalid table credential table name: {}", err))?; - Ok(format!("namespaces/{}/tables/{}", namespace.storage_id(), table.as_str())) -} - -fn table_credential_session_policy(entry: &crate::table_catalog::TableEntry, object_prefix: &str) -> S3Result { - let bucket = &entry.table_bucket; - let object_prefix = normalize_table_credential_object_prefix(object_prefix)?; - let catalog_resource = table_credential_catalog_resource(entry)?; - let policy = serde_json::json!({ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject", - "s3:AbortMultipartUpload", - "s3:ListMultipartUploadParts" - ], - "Resource": [ - format!("arn:aws:s3:::{bucket}/{object_prefix}*") - ] - }, - { - "Effect": "Allow", - "Action": [ - "s3:GetBucketLocation" - ], - "Resource": [ - format!("arn:aws:s3:::{bucket}") - ] - }, - { - "Effect": "Allow", - "Action": [ - "admin:GetTableMetadata", - "admin:SetTableMetadata" - ], - "Resource": [ - format!("arn:aws:s3:::{bucket}/{catalog_resource}") - ] - } - ] - }); - let data = serde_json::to_vec(&policy) - .map_err(|err| s3_error!(InternalError, "failed to serialize table credential policy: {}", err))?; - Policy::parse_config(&data).map_err(|err| s3_error!(InvalidRequest, "invalid table credential policy: {}", err)) -} - -fn storage_credential_from_issued(scope: TableCredentialScope, issued: IssuedTableCredentials) -> RestStorageCredential { - let mut config = BTreeMap::new(); - config.insert(S3_ACCESS_KEY_ID_CONFIG_KEY.to_string(), issued.access_key_id); - config.insert(S3_SECRET_ACCESS_KEY_CONFIG_KEY.to_string(), issued.secret_access_key); - config.insert(S3_SESSION_TOKEN_CONFIG_KEY.to_string(), issued.session_token); - config.insert(CREDENTIAL_VENDING_CONFIG_KEY.to_string(), CREDENTIAL_VENDING_SUPPORTED.to_string()); - config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CATALOG_VENDED.to_string()); - config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); - config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), scope.scope_prefix.clone()); - config.insert( - CREDENTIAL_EXPIRATION_CONFIG_KEY.to_string(), - issued.expiration.unix_timestamp().to_string(), - ); - RestStorageCredential { - prefix: scope.scope_prefix, - config, - } -} - -fn table_metadata_location_for_client(table_bucket: &str, metadata_location: &str) -> String { - if crate::table_catalog::is_reserved_table_object_key(metadata_location) { - format!("s3://{table_bucket}/{metadata_location}") - } else { - metadata_location.to_string() - } -} - -fn table_metadata_location_for_catalog(table_bucket: &str, metadata_location: &str) -> S3Result { - crate::table_catalog::table_catalog_object_key_from_location(table_bucket, metadata_location) - .ok_or_else(|| s3_error!(InvalidRequest, "metadata location must be inside the table bucket")) -} - -fn table_metadata_for_client(table_bucket: &str, mut metadata: serde_json::Value) -> serde_json::Value { - if let Some(metadata_log) = metadata.get_mut("metadata-log").and_then(serde_json::Value::as_array_mut) { - for entry in metadata_log { - let Some(metadata_file) = entry.get_mut("metadata-file") else { - continue; - }; - let Some(metadata_location) = metadata_file.as_str() else { - continue; - }; - if crate::table_catalog::is_reserved_table_object_key(metadata_location) { - *metadata_file = serde_json::Value::String(table_metadata_location_for_client(table_bucket, metadata_location)); - } - } - } - - metadata -} - -fn load_table_response_from_entry(entry: crate::table_catalog::TableEntry, metadata: serde_json::Value) -> RestLoadTableResponse { - let mut config = BTreeMap::new(); - let warehouse_location = entry.warehouse_location.clone(); - let metadata_location = table_metadata_location_for_client(&entry.table_bucket, &entry.metadata_location); - let metadata = table_metadata_for_client(&entry.table_bucket, metadata); - config.insert("warehouse-location".to_string(), warehouse_location.clone()); - config.insert(CREDENTIAL_VENDING_CONFIG_KEY.to_string(), CREDENTIAL_VENDING_UNSUPPORTED.to_string()); - config.insert( - CREDENTIAL_VENDING_REASON_CONFIG_KEY.to_string(), - CREDENTIAL_VENDING_UNSUPPORTED_REASON.to_string(), - ); - config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); - config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), warehouse_location); - config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()); - - RestLoadTableResponse { - metadata_location, - metadata, - config, - storage_credentials: Vec::new(), - } -} - -fn load_view_response_from_entry(entry: crate::table_catalog::ViewEntry, metadata: serde_json::Value) -> RestLoadViewResponse { - let mut config = BTreeMap::new(); - let warehouse_location = entry.warehouse_location.clone(); - config.insert("warehouse-location".to_string(), warehouse_location.clone()); - config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); - config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), warehouse_location); - config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()); - - RestLoadViewResponse { - metadata_location: entry.metadata_location, - metadata, - config, - } -} - -fn load_credentials_response_config(vending: &str, mode: &str, reason: Option<&str>) -> BTreeMap { - let mut config = BTreeMap::new(); - config.insert(CREDENTIAL_VENDING_CONFIG_KEY.to_string(), vending.to_string()); - config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), mode.to_string()); - if let Some(reason) = reason { - config.insert(CREDENTIAL_VENDING_REASON_CONFIG_KEY.to_string(), reason.to_string()); - } - config -} - -fn add_table_credential_scope_config(config: &mut BTreeMap, scope_prefix: &str) { - config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); - config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), scope_prefix.to_string()); -} - -async fn load_credentials_response_from_entry( - entry: &crate::table_catalog::TableEntry, - issuer: &dyn TableCredentialIssuer, - principal: Option<&rustfs_credentials::Credentials>, -) -> S3Result { - if !issuer.enabled() { - return Ok(RestLoadCredentialsResponse { - config: load_credentials_response_config( - CREDENTIAL_VENDING_UNSUPPORTED, - CREDENTIAL_MODE_CLIENT_PROVIDED, - Some(CREDENTIAL_VENDING_DISABLED_REASON), - ), - storage_credentials: Vec::new(), - }); - } - let scope = table_credential_scope(entry)?; - let request = TableCredentialIssueRequest { - entry, - principal, - scope_prefix: scope.scope_prefix.clone(), - object_prefix: scope.object_prefix.clone(), - }; - let scope_prefix = scope.scope_prefix.clone(); - let storage_credentials = match issuer.issue_table_credentials(request).await? { - Some(issued) => vec![storage_credential_from_issued(scope, issued)], - None => { - let mut config = load_credentials_response_config( - CREDENTIAL_VENDING_UNSUPPORTED, - CREDENTIAL_MODE_CLIENT_PROVIDED, - Some(CREDENTIAL_VENDING_UNSUPPORTED_REASON), - ); - add_table_credential_scope_config(&mut config, &scope_prefix); - return Ok(RestLoadCredentialsResponse { - config, - storage_credentials: Vec::new(), - }); - } - }; - let mut config = load_credentials_response_config(CREDENTIAL_VENDING_SUPPORTED, CREDENTIAL_MODE_CATALOG_VENDED, None); - add_table_credential_scope_config(&mut config, &scope_prefix); - Ok(RestLoadCredentialsResponse { - config, - storage_credentials, - }) -} - -fn commit_table_response_from_result( - result: crate::table_catalog::TableCommitResult, - metadata: serde_json::Value, -) -> RestCommitTableResponse { - let metadata_location = table_metadata_location_for_client(&result.table.table_bucket, &result.table.metadata_location); - let metadata = table_metadata_for_client(&result.table.table_bucket, metadata); - RestCommitTableResponse { - metadata_location, - metadata, - version_token: result.table.version_token, - generation: result.table.generation, - commit_id: result.commit_log.commit_id, - } -} - -fn table_metadata_location_response_from_entry(entry: crate::table_catalog::TableEntry) -> TableMetadataLocationResponse { - let metadata_location = table_metadata_location_for_client(&entry.table_bucket, &entry.metadata_location); - TableMetadataLocationResponse { - metadata_location, - version_token: entry.version_token, - generation: entry.generation, - warehouse_location: entry.warehouse_location, - } -} - -fn table_commit_request_from_rest_request( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: RestCommitTableRequest, -) -> S3Result { - let expected_metadata_location = request - .expected_metadata_location - .ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires expected-metadata-location"))?; - let new_metadata_location = request - .new_metadata_location - .ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires new-metadata-location"))?; - Ok(crate::table_catalog::TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - commit_id: request.commit_id.unwrap_or_else(|| Uuid::new_v4().to_string()), - idempotency_key: request.idempotency_key, - operation: request.operation.unwrap_or_else(|| "commit".to_string()), - expected_version_token: request - .expected_version_token - .ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires expected-version-token"))?, - expected_metadata_location: table_metadata_location_for_catalog(bucket, &expected_metadata_location)?, - new_metadata_location: table_metadata_location_for_catalog(bucket, &new_metadata_location)?, - requirements: request.requirements, - writer: request.writer, - }) -} - -fn validate_table_location_in_bucket(bucket: &str, location: &str) -> S3Result<()> { - crate::table_catalog::validate_table_warehouse_location(bucket, location).map_err(catalog_store_error) -} - -fn validate_view_location_in_bucket(bucket: &str, location: &str) -> S3Result<()> { - crate::table_catalog::validate_view_warehouse_location(bucket, location).map_err(catalog_store_error) -} - -fn metadata_table_uuid(metadata: &serde_json::Value) -> S3Result<&str> { - metadata - .get("table-uuid") - .and_then(serde_json::Value::as_str) - .filter(|uuid| !uuid.is_empty()) - .ok_or_else(|| s3_error!(InvalidRequest, "table metadata is missing table-uuid")) -} - -fn metadata_format_version(metadata: &serde_json::Value) -> S3Result { - let version = metadata - .get("format-version") - .and_then(serde_json::Value::as_u64) - .filter(|version| *version > 0) - .ok_or_else(|| s3_error!(InvalidRequest, "table metadata is missing format-version"))?; - u16::try_from(version).map_err(|_| s3_error!(InvalidRequest, "table metadata format-version is too large")) -} - -fn metadata_table_location(metadata: &serde_json::Value) -> S3Result<&str> { - metadata - .get("location") - .and_then(serde_json::Value::as_str) - .filter(|location| !location.is_empty()) - .ok_or_else(|| s3_error!(InvalidRequest, "table metadata is missing location")) -} - -fn validate_metadata_table_location_in_bucket(bucket: &str, metadata: &serde_json::Value) -> S3Result<()> { - let location = metadata_table_location(metadata)?; - validate_table_location_in_bucket(bucket, location) -} - -fn validate_metadata_view_location_in_bucket(bucket: &str, metadata: &serde_json::Value) -> S3Result<()> { - let location = metadata_table_location(metadata)?; - validate_view_location_in_bucket(bucket, location) -} - -fn validate_metadata_matches_current_metadata( - current_metadata: &serde_json::Value, - target_metadata: &serde_json::Value, -) -> S3Result<()> { - let expected_table_uuid = metadata_table_uuid(current_metadata)?; - metadata_format_version(current_metadata)?; - let target_table_uuid = metadata_table_uuid(target_metadata)?; - metadata_format_version(target_metadata)?; - if target_table_uuid != expected_table_uuid { - return Err(s3_error!( - InvalidRequest, - "table metadata table-uuid does not match current table metadata" - )); - } - Ok(()) -} - -fn metadata_view_uuid(metadata: &serde_json::Value) -> S3Result<&str> { - metadata - .get("view-uuid") - .and_then(serde_json::Value::as_str) - .filter(|uuid| !uuid.is_empty()) - .ok_or_else(|| s3_error!(InvalidRequest, "view metadata is missing view-uuid")) -} - -fn validate_metadata_matches_current_view_metadata( - current_metadata: &serde_json::Value, - target_metadata: &serde_json::Value, -) -> S3Result<()> { - let expected_view_uuid = metadata_view_uuid(current_metadata)?; - metadata_format_version(current_metadata)?; - let target_view_uuid = metadata_view_uuid(target_metadata)?; - metadata_format_version(target_metadata)?; - if target_view_uuid != expected_view_uuid { - return Err(s3_error!(InvalidRequest, "view metadata view-uuid does not match current view metadata")); - } - Ok(()) -} - -fn adopt_registered_metadata_identity( - entry: &mut crate::table_catalog::TableEntry, - metadata: &serde_json::Value, -) -> S3Result<()> { - entry.table_uuid = metadata_table_uuid(metadata)?.to_string(); - entry.format_version = metadata_format_version(metadata)?; - entry.warehouse_location = metadata_table_location(metadata)?.to_string(); - Ok(()) -} - -fn table_entry_from_register_request( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - request: RegisterTableRequest, -) -> S3Result { - if request.overwrite { - return Err(s3_error!(NotImplemented, "register table overwrite is not supported")); - } - let table = crate::table_catalog::IdentifierSegment::parse(request.name) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; - if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, &metadata_location) { - return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); - } - - let table_id = Uuid::new_v4().to_string(); - Ok(crate::table_catalog::TableEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: table_id.clone(), - table_uuid: Uuid::new_v4().to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: format!("s3://{bucket}/tables/{table_id}"), - metadata_location, - version_token: format!("token-{}", Uuid::new_v4()), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }) -} - -fn table_entry_from_import_request( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: CatalogImportRequest, -) -> S3Result { - let table = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; - if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, &metadata_location) { - return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); - } - - let table_id = Uuid::new_v4().to_string(); - Ok(crate::table_catalog::TableEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: table_id.clone(), - table_uuid: Uuid::new_v4().to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: format!("s3://{bucket}/tables/{table_id}"), - metadata_location, - version_token: format!("token-{}", Uuid::new_v4()), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: request.properties, - created_at: None, - updated_at: None, - }) -} - -fn table_entry_from_create_table_request( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - request: CreateTableRequest, -) -> S3Result<(crate::table_catalog::TableEntry, serde_json::Value)> { - if request.stage_create { - return Err(s3_error!(NotImplemented, "stage-create is not supported")); - } - - let table = crate::table_catalog::IdentifierSegment::parse(request.name) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - let table_id = Uuid::new_v4().to_string(); - let table_uuid = Uuid::new_v4().to_string(); - let warehouse_location = request.location.unwrap_or_else(|| format!("s3://{bucket}/tables/{table_id}")); - validate_table_location_in_bucket(bucket, &warehouse_location)?; - let metadata_location = - crate::table_catalog::default_table_metadata_file_path(namespace, &table, &next_metadata_file_name(1, &table_id)); - - let mut entry = crate::table_catalog::TableEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id, - table_uuid, - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location, - metadata_location, - version_token: format!("token-{}", Uuid::new_v4()), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: request.properties, - created_at: None, - updated_at: None, - }; - let metadata = initial_table_metadata_json( - &entry, - request.schema, - request.partition_spec, - request.write_order, - entry.properties.clone(), - )?; - entry.format_version = metadata - .get("format-version") - .and_then(serde_json::Value::as_u64) - .and_then(|version| u16::try_from(version).ok()) - .unwrap_or(2); - Ok((entry, metadata)) -} - -fn view_entry_from_create_view_request( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - request: CreateViewRequest, -) -> S3Result<(crate::table_catalog::ViewEntry, serde_json::Value)> { - let view = crate::table_catalog::IdentifierSegment::parse(request.name) - .map_err(|err| s3_error!(InvalidRequest, "invalid view name: {}", err))?; - let view_id = Uuid::new_v4().to_string(); - let view_uuid = Uuid::new_v4().to_string(); - let warehouse_location = request.location.unwrap_or_else(|| format!("s3://{bucket}/views/{view_id}")); - validate_view_location_in_bucket(bucket, &warehouse_location)?; - let metadata_location = - crate::table_catalog::default_view_metadata_file_path(namespace, &view, &next_metadata_file_name(1, &view_id)); - - let entry = crate::table_catalog::ViewEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.as_str().to_string(), - view_id, - view_uuid, - format: "ICEBERG_VIEW".to_string(), - format_version: 1, - warehouse_location, - metadata_location, - version_token: format!("token-{}", Uuid::new_v4()), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: request.properties, - created_at: None, - updated_at: None, - }; - let metadata = initial_view_metadata_json(&entry, request.schema, request.view_version, entry.properties.clone())?; - Ok((entry, metadata)) -} - -fn initial_table_metadata_json( - entry: &crate::table_catalog::TableEntry, - mut schema: serde_json::Value, - partition_spec: Option, - write_order: Option, - properties: BTreeMap, -) -> S3Result { - let schema_object = schema - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "schema must be a JSON object"))?; - schema_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - let schema_id = schema_object - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "schema-id must be an integer"))?; - let last_column_id = max_field_id(&schema); - - let mut spec = partition_spec.unwrap_or_else(|| { - serde_json::json!({ - "spec-id": 0, - "fields": [] - }) - }); - let spec_object = spec - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "partition-spec must be a JSON object"))?; - spec_object - .entry("spec-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - spec_object - .entry("fields".to_string()) - .or_insert_with(|| serde_json::Value::Array(Vec::new())); - let spec_id = spec_object - .get("spec-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "partition spec-id must be an integer"))?; - let last_partition_id = max_partition_field_id(&spec); - - let mut sort_order = write_order.unwrap_or_else(|| { - serde_json::json!({ - "order-id": 0, - "fields": [] - }) - }); - let sort_order_object = sort_order - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "write-order must be a JSON object"))?; - sort_order_object - .entry("order-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - sort_order_object - .entry("fields".to_string()) - .or_insert_with(|| serde_json::Value::Array(Vec::new())); - let sort_order_id = sort_order_object - .get("order-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "sort order-id must be an integer"))?; - - Ok(serde_json::json!({ - "format-version": entry.format_version, - "table-uuid": entry.table_uuid, - "location": entry.warehouse_location, - "last-sequence-number": 0, - "last-updated-ms": current_time_millis(), - "last-column-id": last_column_id, - "schemas": [schema], - "current-schema-id": schema_id, - "partition-specs": [spec], - "default-spec-id": spec_id, - "last-partition-id": last_partition_id, - "sort-orders": [sort_order], - "default-sort-order-id": sort_order_id, - "properties": properties, - "snapshots": [], - "snapshot-log": [], - "metadata-log": [], - "refs": {} - })) -} - -fn initial_view_metadata_json( - entry: &crate::table_catalog::ViewEntry, - mut schema: serde_json::Value, - mut view_version: serde_json::Value, - properties: BTreeMap, -) -> S3Result { - let schema_object = schema - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "schema must be a JSON object"))?; - schema_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - let schema_id = schema_object - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "schema-id must be an integer"))?; - - let view_version_object = view_version - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))?; - view_version_object - .entry("version-id".to_string()) - .or_insert_with(|| serde_json::Value::from(1)); - view_version_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(schema_id)); - view_version_object - .entry("timestamp-ms".to_string()) - .or_insert_with(|| serde_json::Value::from(current_time_millis())); - let version_id = view_version_object - .get("version-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "view-version version-id must be an integer"))?; - let timestamp_ms = view_version_object - .get("timestamp-ms") - .and_then(serde_json::Value::as_i64) - .unwrap_or_else(current_time_millis); - - Ok(serde_json::json!({ - "format-version": entry.format_version, - "view-uuid": entry.view_uuid, - "location": entry.warehouse_location, - "current-version-id": version_id, - "schemas": [schema], - "versions": [view_version], - "version-log": [{ - "timestamp-ms": timestamp_ms, - "version-id": version_id - }], - "metadata-log": [], - "properties": properties - })) -} - -fn current_time_millis() -> i64 { - let now = OffsetDateTime::now_utc(); - now.unix_timestamp() - .saturating_mul(1000) - .saturating_add(i64::from(now.millisecond())) -} - -fn max_field_id(value: &serde_json::Value) -> i64 { - let mut max_id = 0; - collect_max_field_id(value, &mut max_id); - max_id -} - -fn collect_max_field_id(value: &serde_json::Value, max_id: &mut i64) { - match value { - serde_json::Value::Object(object) => { - if let Some(id) = object.get("id").and_then(serde_json::Value::as_i64) { - *max_id = (*max_id).max(id); - } - for child in object.values() { - collect_max_field_id(child, max_id); - } - } - serde_json::Value::Array(values) => { - for child in values { - collect_max_field_id(child, max_id); - } - } - _ => {} - } -} - -fn max_partition_field_id(value: &serde_json::Value) -> i64 { - let mut max_id = 999; - let Some(fields) = value.get("fields").and_then(serde_json::Value::as_array) else { - return max_id; - }; - for field in fields { - if let Some(field_id) = field.get("field-id").and_then(serde_json::Value::as_i64) { - max_id = max_id.max(field_id); - } - } - max_id -} - -fn standard_commit_ids(commit_id: Option) -> (String, String) { - match commit_id { - Some(commit_id) => match Uuid::parse_str(&commit_id) { - Ok(uuid) => { - let commit_id = uuid.to_string(); - (commit_id.clone(), commit_id) - } - Err(_) => { - let metadata_file_token = table_catalog_path_hash(&commit_id); - (commit_id, metadata_file_token) - } - }, - None => { - let commit_id = Uuid::new_v4().to_string(); - (commit_id.clone(), commit_id) - } - } -} - -fn next_metadata_file_name(generation: u64, metadata_file_token: &str) -> String { - format!("{generation:05}-{metadata_file_token}.metadata.json") -} - -fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements: &[serde_json::Value]) -> S3Result<()> { - for requirement in requirements { - let requirement_type = requirement - .get("type") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "commit requirement type is required"))?; - match requirement_type { - "assert-create" => { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: table already exists")); - } - "assert-table-uuid" => { - let expected = requirement - .get("uuid") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "assert-table-uuid requires uuid"))?; - let actual = metadata - .get("table-uuid") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing table-uuid"))?; - if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: table uuid changed")); - } - } - "assert-current-schema-id" => { - validate_i64_requirement(metadata, requirement, "current-schema-id", "current schema id")?; - } - "assert-default-spec-id" => { - validate_i64_requirement(metadata, requirement, "default-spec-id", "default spec id")?; - } - "assert-default-sort-order-id" => { - validate_i64_requirement(metadata, requirement, "default-sort-order-id", "default sort order id")?; - } - "assert-last-assigned-field-id" => { - validate_i64_requirement_with_metadata_key( - metadata, - requirement, - "last-assigned-field-id", - "last-column-id", - "last assigned field id", - )?; - } - "assert-last-assigned-partition-id" => { - validate_i64_requirement_with_metadata_key( - metadata, - requirement, - "last-assigned-partition-id", - "last-partition-id", - "last assigned partition id", - )?; - } - "assert-ref-snapshot-id" => validate_ref_snapshot_requirement(metadata, requirement)?, - "assert-current-snapshot-id" => validate_current_snapshot_requirement(metadata, requirement)?, - _ => return Err(s3_error!(NotImplemented, "unsupported commit requirement: {requirement_type}")), - } - } - Ok(()) -} - -fn validate_i64_requirement( - metadata: &serde_json::Value, - requirement: &serde_json::Value, - key: &str, - label: &str, -) -> S3Result<()> { - validate_i64_requirement_with_metadata_key(metadata, requirement, key, key, label) -} - -fn validate_i64_requirement_with_metadata_key( - metadata: &serde_json::Value, - requirement: &serde_json::Value, - requirement_key: &str, - metadata_key: &str, - label: &str, -) -> S3Result<()> { - let expected = requirement - .get(requirement_key) - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "{requirement_key} must be an integer"))?; - let actual = metadata - .get(metadata_key) - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing {metadata_key}"))?; - if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: {label} changed")); - } - Ok(()) -} - -fn validate_ref_snapshot_requirement(metadata: &serde_json::Value, requirement: &serde_json::Value) -> S3Result<()> { - let ref_name = requirement - .get("ref") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "assert-ref-snapshot-id requires ref"))?; - let refs = metadata.get("refs").and_then(serde_json::Value::as_object); - let actual = refs - .and_then(|refs| refs.get(ref_name)) - .and_then(|reference| reference.get("snapshot-id")) - .and_then(serde_json::Value::as_i64); - if requirement.get("snapshot-id").is_some_and(serde_json::Value::is_null) { - if actual.is_some() { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: snapshot ref exists")); - } - return Ok(()); - } - let expected = requirement - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "assert-ref-snapshot-id requires snapshot-id"))?; - if actual != Some(expected) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: snapshot ref changed")); - } - Ok(()) -} - -fn validate_current_snapshot_requirement(metadata: &serde_json::Value, requirement: &serde_json::Value) -> S3Result<()> { - let actual = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); - if requirement.get("snapshot-id").is_some_and(serde_json::Value::is_null) { - if actual.is_some() { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: current snapshot exists")); - } - return Ok(()); - } - let expected = requirement - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "assert-current-snapshot-id requires snapshot-id"))?; - if actual != Some(expected) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: current snapshot changed")); - } - Ok(()) -} - -fn apply_table_commit_updates( - mut metadata: serde_json::Value, - updates: &[serde_json::Value], - previous_metadata_location: &str, -) -> S3Result { - if !metadata.is_object() { - return Err(s3_error!(InvalidRequest, "current table metadata must be a JSON object")); - } - - for update in updates { - let action = update - .get("action") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "table update action is required"))?; - match action { - "assign-uuid" => apply_assign_uuid_update(&mut metadata, update)?, - "upgrade-format-version" => apply_upgrade_format_version_update(&mut metadata, update)?, - "add-schema" => apply_add_schema_update(&mut metadata, update)?, - "set-current-schema" => apply_set_current_schema_update(&mut metadata, update)?, - "add-spec" => apply_add_spec_update(&mut metadata, update)?, - "set-default-spec" => apply_set_default_spec_update(&mut metadata, update)?, - "add-sort-order" => apply_add_sort_order_update(&mut metadata, update)?, - "set-default-sort-order" => apply_set_default_sort_order_update(&mut metadata, update)?, - "add-snapshot" => apply_add_snapshot_update(&mut metadata, update)?, - "set-snapshot-ref" => apply_set_snapshot_ref_update(&mut metadata, update)?, - "remove-snapshots" => apply_remove_snapshots_update(&mut metadata, update)?, - "remove-snapshot-ref" => apply_remove_snapshot_ref_update(&mut metadata, update)?, - "set-location" => apply_set_location_update(&mut metadata, update)?, - "set-properties" => apply_set_properties_update(&mut metadata, update)?, - "remove-properties" => apply_remove_properties_update(&mut metadata, update)?, - _ => return Err(s3_error!(NotImplemented, "unsupported table update: {action}")), - } - } - - append_previous_metadata_log(&mut metadata, previous_metadata_location)?; - metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(current_time_millis())); - Ok(metadata) -} - -fn validate_view_commit_requirements(metadata: &serde_json::Value, requirements: &[serde_json::Value]) -> S3Result<()> { - for requirement in requirements { - let requirement_type = requirement - .get("type") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "commit requirement type is required"))?; - match requirement_type { - "assert-view-uuid" => { - let expected = requirement - .get("uuid") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "assert-view-uuid requires uuid"))?; - let actual = metadata - .get("view-uuid") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "current view metadata is missing view-uuid"))?; - if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: view uuid changed")); - } - } - "assert-current-view-version-id" => { - validate_i64_requirement_with_metadata_key( - metadata, - requirement, - "current-view-version-id", - "current-version-id", - "current view version id", - )?; - } - _ => return Err(s3_error!(NotImplemented, "unsupported view commit requirement: {requirement_type}")), - } - } - Ok(()) -} - -fn apply_view_commit_updates( - mut metadata: serde_json::Value, - updates: &[serde_json::Value], - previous_metadata_location: &str, -) -> S3Result { - if !metadata.is_object() { - return Err(s3_error!(InvalidRequest, "current view metadata must be a JSON object")); - } - - for update in updates { - let action = update - .get("action") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "view update action is required"))?; - match action { - "assign-uuid" => apply_assign_uuid_update(&mut metadata, update)?, - "add-schema" => apply_add_schema_update(&mut metadata, update)?, - "set-current-schema" => apply_set_current_schema_update(&mut metadata, update)?, - "add-view-version" => apply_add_view_version_update(&mut metadata, update)?, - "set-current-view-version" => apply_set_current_view_version_update(&mut metadata, update)?, - "set-location" => apply_set_location_update(&mut metadata, update)?, - "set-properties" => apply_set_properties_update(&mut metadata, update)?, - "remove-properties" => apply_remove_properties_update(&mut metadata, update)?, - _ => return Err(s3_error!(NotImplemented, "unsupported view update: {action}")), - } - } - - append_previous_metadata_log(&mut metadata, previous_metadata_location)?; - metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(current_time_millis())); - Ok(metadata) -} - -fn apply_assign_uuid_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let uuid = update - .get("uuid") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "assign-uuid requires uuid"))?; - let object = metadata_object_mut(metadata)?; - if let Some(existing) = object.get("table-uuid").and_then(serde_json::Value::as_str) - && existing != uuid - { - return Err(s3_error!(PreconditionFailed, "cannot reassign table uuid")); - } - object.insert("table-uuid".to_string(), serde_json::Value::String(uuid.to_string())); - Ok(()) -} - -fn apply_add_view_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let mut view_version = update - .get("view-version") - .cloned() - .ok_or_else(|| s3_error!(InvalidRequest, "add-view-version requires view-version"))?; - if !view_version.is_object() { - return Err(s3_error!(InvalidRequest, "view-version must be a JSON object")); - } - if view_version.get("version-id").is_none() { - let next_id = next_array_object_i64(metadata, "versions", "version-id")?; - view_version - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))? - .insert("version-id".to_string(), serde_json::Value::from(next_id)); - } - view_version - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))? - .entry("timestamp-ms".to_string()) - .or_insert_with(|| serde_json::Value::from(current_time_millis())); - ensure_array_field(metadata, "versions")?.push(view_version); - Ok(()) -} - -fn apply_set_current_view_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let requested_id = update - .get("view-version-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "set-current-view-version requires view-version-id"))?; - let version_id = if requested_id == -1 { - last_array_object_i64(metadata, "versions", "version-id")? - } else { - requested_id - }; - metadata_object_mut(metadata)?.insert("current-version-id".to_string(), serde_json::Value::from(version_id)); - ensure_array_field(metadata, "version-log")?.push(serde_json::json!({ - "timestamp-ms": current_time_millis(), - "version-id": version_id - })); - Ok(()) -} - -fn apply_upgrade_format_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let version = update - .get("format-version") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "upgrade-format-version requires format-version"))?; - let current = metadata - .get("format-version") - .and_then(serde_json::Value::as_i64) - .unwrap_or_default(); - if version < current { - return Err(s3_error!(InvalidRequest, "format-version cannot be downgraded")); - } - metadata_object_mut(metadata)?.insert("format-version".to_string(), serde_json::Value::from(version)); - Ok(()) -} - -fn apply_add_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let mut schema = update - .get("schema") - .cloned() - .ok_or_else(|| s3_error!(InvalidRequest, "add-schema requires schema"))?; - if !schema.is_object() { - return Err(s3_error!(InvalidRequest, "add-schema schema must be a JSON object")); - } - if schema.get("schema-id").is_none() { - let next_id = next_array_object_i64(metadata, "schemas", "schema-id")?; - schema - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-schema schema must be a JSON object"))? - .insert("schema-id".to_string(), serde_json::Value::from(next_id)); - } - let last_column_id = max_field_id(&schema); - ensure_array_field(metadata, "schemas")?.push(schema); - let object = metadata_object_mut(metadata)?; - let current_last = object - .get("last-column-id") - .and_then(serde_json::Value::as_i64) - .unwrap_or_default(); - object.insert("last-column-id".to_string(), serde_json::Value::from(current_last.max(last_column_id))); - Ok(()) -} - -fn apply_set_current_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let requested_id = update - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "set-current-schema requires schema-id"))?; - let schema_id = if requested_id == -1 { - last_array_object_i64(metadata, "schemas", "schema-id")? - } else { - requested_id - }; - metadata_object_mut(metadata)?.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); - Ok(()) -} - -fn apply_add_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let mut spec = update - .get("spec") - .cloned() - .ok_or_else(|| s3_error!(InvalidRequest, "add-spec requires spec"))?; - if !spec.is_object() { - return Err(s3_error!(InvalidRequest, "add-spec spec must be a JSON object")); - } - if spec.get("spec-id").is_none() { - let next_id = next_array_object_i64(metadata, "partition-specs", "spec-id")?; - spec.as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-spec spec must be a JSON object"))? - .insert("spec-id".to_string(), serde_json::Value::from(next_id)); - } - let last_partition_id = max_partition_field_id(&spec); - ensure_array_field(metadata, "partition-specs")?.push(spec); - let object = metadata_object_mut(metadata)?; - let current_last = object - .get("last-partition-id") - .and_then(serde_json::Value::as_i64) - .unwrap_or(999); - object.insert( - "last-partition-id".to_string(), - serde_json::Value::from(current_last.max(last_partition_id)), - ); - Ok(()) -} - -fn apply_set_default_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let requested_id = update - .get("spec-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "set-default-spec requires spec-id"))?; - let spec_id = if requested_id == -1 { - last_array_object_i64(metadata, "partition-specs", "spec-id")? - } else { - requested_id - }; - metadata_object_mut(metadata)?.insert("default-spec-id".to_string(), serde_json::Value::from(spec_id)); - Ok(()) -} - -fn apply_add_sort_order_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let mut sort_order = update - .get("sort-order") - .cloned() - .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order requires sort-order"))?; - if !sort_order.is_object() { - return Err(s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object")); - } - if sort_order.get("order-id").is_none() { - let next_id = next_array_object_i64(metadata, "sort-orders", "order-id")?; - sort_order - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object"))? - .insert("order-id".to_string(), serde_json::Value::from(next_id)); - } - ensure_array_field(metadata, "sort-orders")?.push(sort_order); - Ok(()) -} - -fn apply_set_default_sort_order_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let requested_id = update - .get("sort-order-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "set-default-sort-order requires sort-order-id"))?; - let sort_order_id = if requested_id == -1 { - last_array_object_i64(metadata, "sort-orders", "order-id")? - } else { - requested_id - }; - metadata_object_mut(metadata)?.insert("default-sort-order-id".to_string(), serde_json::Value::from(sort_order_id)); - Ok(()) -} - -fn apply_add_snapshot_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let snapshot = update - .get("snapshot") - .cloned() - .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?; - let snapshot_id = snapshot - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-id must be an integer"))?; - let sequence_number = snapshot - .get("sequence-number") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer"))?; - let timestamp_ms = snapshot - .get("timestamp-ms") - .and_then(serde_json::Value::as_i64) - .unwrap_or_else(current_time_millis); - validate_added_snapshot(metadata, &snapshot, snapshot_id, sequence_number)?; - ensure_array_field(metadata, "snapshots")?.push(snapshot); - let object = metadata_object_mut(metadata)?; - object.insert("last-sequence-number".to_string(), serde_json::Value::from(sequence_number)); - object.insert("current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id)); - ensure_array_field(metadata, "snapshot-log")?.push(serde_json::json!({ - "timestamp-ms": timestamp_ms, - "snapshot-id": snapshot_id - })); - Ok(()) -} - -fn validate_added_snapshot( - metadata: &serde_json::Value, - snapshot: &serde_json::Value, - snapshot_id: i64, - sequence_number: i64, -) -> S3Result<()> { - if metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .is_some_and(|snapshots| { - snapshots - .iter() - .any(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) - }) - { - return Err(s3_error!(PreconditionFailed, "snapshot id already exists")); - } - - let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); - if let Some(parent_snapshot_id) = snapshot.get("parent-snapshot-id").and_then(serde_json::Value::as_i64) - && Some(parent_snapshot_id) != current_snapshot_id - { - return Err(s3_error!(PreconditionFailed, "snapshot parent no longer matches current snapshot")); - } - - let current_sequence_number = metadata - .get("last-sequence-number") - .and_then(serde_json::Value::as_i64) - .unwrap_or_default(); - if sequence_number <= current_sequence_number { - return Err(s3_error!(PreconditionFailed, "snapshot sequence number must advance")); - } - - if !snapshot_has_manifest_references(snapshot) { - return Err(s3_error!(InvalidRequest, "snapshot manifest-list or manifests are required")); - } - - let operation = snapshot - .get("summary") - .and_then(|summary| summary.get("operation")) - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot summary.operation is required"))?; - if !matches!(operation, "append" | "overwrite" | "delete" | "replace") { - return Err(s3_error!(NotImplemented, "unsupported snapshot operation: {operation}")); - } - - Ok(()) -} - -fn snapshot_has_manifest_references(snapshot: &serde_json::Value) -> bool { - if snapshot - .get("manifest-list") - .and_then(serde_json::Value::as_str) - .is_some_and(|manifest_list| !manifest_list.is_empty()) - { - return true; - } - snapshot - .get("manifests") - .and_then(serde_json::Value::as_array) - .is_some_and(|manifests| { - !manifests.is_empty() - && manifests - .iter() - .all(|manifest| manifest.as_str().is_some_and(|manifest| !manifest.is_empty())) - }) -} - -#[derive(Default)] -struct SnapshotLiveFiles { - data_files: BTreeMap, - delete_files: BTreeMap, - manifest_files: BTreeMap, -} - -impl SnapshotLiveFiles { - fn contains(&self, location: &str) -> bool { - self.data_files.contains_key(location) || self.delete_files.contains_key(location) - } - - fn identity( - &self, - location: &str, - object_kind: &crate::table_catalog::TableMetadataMaintenanceObjectKind, - ) -> Option<&SnapshotFileIdentity> { - match object_kind { - crate::table_catalog::TableMetadataMaintenanceObjectKind::DataFile => self.data_files.get(location), - crate::table_catalog::TableMetadataMaintenanceObjectKind::DeleteFile => self.delete_files.get(location), - _ => None, - } - } -} - -#[derive(Default)] -struct SnapshotFileChanges { - added_data_files: BTreeSet, - added_delete_files: BTreeSet, - deleted_data_files: BTreeSet, - deleted_delete_files: BTreeSet, -} - -impl SnapshotFileChanges { - fn has_delete_or_row_level_change(&self) -> bool { - !self.added_delete_files.is_empty() || !self.deleted_data_files.is_empty() || !self.deleted_delete_files.is_empty() - } - - fn has_any_change(&self) -> bool { - !self.added_data_files.is_empty() || !self.added_delete_files.is_empty() || self.has_deleted_files() - } - - fn has_deleted_files(&self) -> bool { - !self.deleted_data_files.is_empty() || !self.deleted_delete_files.is_empty() - } -} - -struct SnapshotChangeContext<'a> { - current_live_files: &'a SnapshotLiveFiles, - snapshot_id: i64, - sequence_number: i64, -} - -#[derive(PartialEq, Eq)] -struct SnapshotManifestIdentity { - sequence_number: Option, - added_snapshot_id: Option, -} - -#[derive(PartialEq, Eq)] -struct SnapshotFileIdentity { - sequence_number: Option, - file_sequence_number: Option, -} - -async fn validate_table_snapshot_commit_conflicts( - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &crate::table_catalog::IdentifierSegment, - entry: &crate::table_catalog::TableEntry, - current_metadata: &serde_json::Value, - updates: &[serde_json::Value], -) -> S3Result<()> -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let Some(snapshot) = added_snapshot_update(updates)? else { - return Ok(()); - }; - let snapshot_id = snapshot - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-id must be an integer"))?; - let sequence_number = snapshot - .get("sequence-number") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer"))?; - let operation = snapshot - .get("summary") - .and_then(|summary| summary.get("operation")) - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot summary.operation is required"))?; - - let current_live_files = - load_current_snapshot_live_files(metadata_backend, bucket, namespace, table, entry, current_metadata).await?; - let changes = load_snapshot_file_changes( - metadata_backend, - bucket, - namespace, - table, - entry, - snapshot, - SnapshotChangeContext { - current_live_files: ¤t_live_files, - snapshot_id, - sequence_number, - }, - ) - .await?; - - for location in changes.added_data_files.iter().chain(changes.added_delete_files.iter()) { - if current_live_files.contains(location) { - return Err(s3_error!( - PreconditionFailed, - "commit requirement failed: added file already exists in current snapshot" - )); - } - } - - match operation { - "append" => { - if changes.has_deleted_files() || !changes.added_delete_files.is_empty() { - return Err(s3_error!(InvalidRequest, "append snapshot cannot delete data files or add delete files")); - } - } - "overwrite" | "delete" | "replace" => { - if current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_none() - { - return Err(s3_error!(InvalidRequest, "row-level snapshot operation requires a current snapshot")); - } - if operation == "overwrite" { - if !changes.has_any_change() { - return Err(s3_error!(InvalidRequest, "overwrite snapshot operation requires changed files")); - } - } else if !changes.has_delete_or_row_level_change() { - return Err(s3_error!( - InvalidRequest, - "row-level snapshot operation requires deleted data files or added delete files" - )); - } - for location in changes.deleted_data_files.iter().chain(changes.deleted_delete_files.iter()) { - if !current_live_files.contains(location) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: deleted file is not current")); - } - } - } - _ => return Err(s3_error!(NotImplemented, "unsupported snapshot operation: {operation}")), - } - - Ok(()) -} - -fn added_snapshot_update(updates: &[serde_json::Value]) -> S3Result> { - let mut snapshot = None; - for update in updates { - if update.get("action").and_then(serde_json::Value::as_str) != Some("add-snapshot") { - continue; - } - if snapshot.is_some() { - return Err(s3_error!(InvalidRequest, "standard commit supports one add-snapshot update")); - } - snapshot = Some( - update - .get("snapshot") - .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?, - ); - } - Ok(snapshot) -} - -async fn load_current_snapshot_live_files( - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &crate::table_catalog::IdentifierSegment, - entry: &crate::table_catalog::TableEntry, - current_metadata: &serde_json::Value, -) -> S3Result -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let Some(current_snapshot_id) = current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - else { - return Ok(SnapshotLiveFiles::default()); - }; - let snapshot = current_metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .and_then(|snapshots| { - snapshots - .iter() - .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(current_snapshot_id)) - }) - .ok_or_else(|| s3_error!(InvalidRequest, "current snapshot metadata is missing"))?; - - let mut live_files = SnapshotLiveFiles::default(); - for manifest in read_snapshot_manifest_references(metadata_backend, bucket, namespace, table, entry, snapshot).await? { - let SnapshotManifestLocation { - manifest_path, - sequence_number, - added_snapshot_id, - } = manifest.location; - live_files.manifest_files.insert( - manifest_path, - SnapshotManifestIdentity { - sequence_number, - added_snapshot_id, - }, - ); - for reference in manifest.references { - let status = reference - .entry_status - .ok_or_else(|| s3_error!(InvalidRequest, "manifest entry status is required"))?; - match status { - 0 | 1 => { - let identity = SnapshotFileIdentity { - sequence_number: reference.sequence_number, - file_sequence_number: reference.file_sequence_number, - }; - match reference.object_kind { - crate::table_catalog::TableMetadataMaintenanceObjectKind::DataFile => { - live_files.data_files.insert(reference.location, identity); - } - crate::table_catalog::TableMetadataMaintenanceObjectKind::DeleteFile => { - live_files.delete_files.insert(reference.location, identity); - } - _ => {} - } - } - 2 => {} - _ => return Err(s3_error!(InvalidRequest, "manifest entry status is unsupported")), - } - } - } - Ok(live_files) -} - -async fn load_snapshot_file_changes( - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &crate::table_catalog::IdentifierSegment, - entry: &crate::table_catalog::TableEntry, - snapshot: &serde_json::Value, - context: SnapshotChangeContext<'_>, -) -> S3Result -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let mut changes = SnapshotFileChanges::default(); - for manifest in read_snapshot_manifest_references(metadata_backend, bucket, namespace, table, entry, snapshot).await? { - let inherited_identity = context - .current_live_files - .manifest_files - .get(&manifest.location.manifest_path); - if let Some(inherited_identity) = inherited_identity { - let candidate_identity = SnapshotManifestIdentity { - sequence_number: manifest.location.sequence_number, - added_snapshot_id: manifest.location.added_snapshot_id, - }; - if inherited_identity - .sequence_number - .is_some_and(|sequence_number| candidate_identity.sequence_number != Some(sequence_number)) - || inherited_identity - .added_snapshot_id - .is_some_and(|snapshot_id| candidate_identity.added_snapshot_id != Some(snapshot_id)) - { - return Err(s3_error!(InvalidRequest, "inherited manifest must preserve its manifest-list identity")); - } - continue; - } - if manifest - .location - .added_snapshot_id - .is_some_and(|added_snapshot_id| added_snapshot_id != context.snapshot_id) - { - return Err(s3_error!(InvalidRequest, "new manifest must belong to the committed snapshot")); - } - if manifest - .location - .sequence_number - .is_some_and(|sequence_number| sequence_number != context.sequence_number) - { - return Err(s3_error!(InvalidRequest, "new manifest sequence must match the committed snapshot")); - } - - for reference in manifest.references { - let status = reference - .entry_status - .ok_or_else(|| s3_error!(InvalidRequest, "manifest entry status is required"))?; - if matches!(status, 1 | 2) && reference.snapshot_id != Some(context.snapshot_id) { - return Err(s3_error!( - InvalidRequest, - "manifest changed entries must belong to the committed snapshot" - )); - } - if status == 1 - && manifest.location.sequence_number.is_some_and(|sequence_number| { - reference.sequence_number != Some(sequence_number) || reference.file_sequence_number != Some(sequence_number) - }) - { - return Err(s3_error!(InvalidRequest, "added manifest entry sequence must match its manifest")); - } - if status == 2 - && context - .current_live_files - .identity(&reference.location, &reference.object_kind) - .is_some_and(|current_identity| { - current_identity - != &SnapshotFileIdentity { - sequence_number: reference.sequence_number, - file_sequence_number: reference.file_sequence_number, - } - }) - { - return Err(s3_error!( - InvalidRequest, - "deleted manifest entry must preserve the current file sequence" - )); - } - - match (status, reference.object_kind) { - (0, _) => {} - (1, crate::table_catalog::TableMetadataMaintenanceObjectKind::DataFile) => { - changes.added_data_files.insert(reference.location); - } - (1, crate::table_catalog::TableMetadataMaintenanceObjectKind::DeleteFile) => { - changes.added_delete_files.insert(reference.location); - } - (2, crate::table_catalog::TableMetadataMaintenanceObjectKind::DataFile) => { - changes.deleted_data_files.insert(reference.location); - } - (2, crate::table_catalog::TableMetadataMaintenanceObjectKind::DeleteFile) => { - changes.deleted_delete_files.insert(reference.location); - } - _ => return Err(s3_error!(InvalidRequest, "manifest entry status is unsupported")), - } - } - } - Ok(changes) -} - -struct SnapshotManifestReferences { - location: SnapshotManifestLocation, - references: Vec, -} - -async fn read_snapshot_manifest_references( - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &crate::table_catalog::IdentifierSegment, - entry: &crate::table_catalog::TableEntry, - snapshot: &serde_json::Value, -) -> S3Result> -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let manifest_locations = snapshot_manifest_locations(metadata_backend, bucket, namespace, table, entry, snapshot).await?; - let mut manifests = Vec::new(); - for manifest_location in manifest_locations { - let manifest_key = table_commit_object_key( - bucket, - namespace, - table, - entry, - &manifest_location.manifest_path, - crate::table_catalog::TableMetadataMaintenanceObjectKind::ManifestFile, - )?; - let manifest_object = metadata_backend - .read_object(bucket, &manifest_key) - .await - .map_err(catalog_store_error)? - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot manifest object is missing"))?; - let file_references = - crate::table_catalog::data_file_references_from_manifest_avro(&manifest_object.data).map_err(catalog_store_error)?; - let mut references = Vec::with_capacity(file_references.len()); - for mut reference in file_references { - if reference.snapshot_id.is_none() { - reference.snapshot_id = manifest_location.added_snapshot_id; - } - if reference.sequence_number.is_none() { - reference.sequence_number = manifest_location.sequence_number; - } - if reference.file_sequence_number.is_none() { - reference.file_sequence_number = manifest_location.sequence_number; - } - validate_manifest_data_file_reference(metadata_backend, bucket, namespace, table, entry, &reference).await?; - references.push(reference); - } - manifests.push(SnapshotManifestReferences { - location: manifest_location, - references, - }); - } - Ok(manifests) -} - -#[derive(Debug)] -struct SnapshotManifestLocation { - manifest_path: String, - sequence_number: Option, - added_snapshot_id: Option, -} - -async fn snapshot_manifest_locations( - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &crate::table_catalog::IdentifierSegment, - entry: &crate::table_catalog::TableEntry, - snapshot: &serde_json::Value, -) -> S3Result> -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - if let Some(manifest_list_location) = snapshot.get("manifest-list").and_then(serde_json::Value::as_str) { - let manifest_list_key = table_commit_object_key( - bucket, - namespace, - table, - entry, - manifest_list_location, - crate::table_catalog::TableMetadataMaintenanceObjectKind::ManifestList, - )?; - let manifest_list_object = metadata_backend - .read_object(bucket, &manifest_list_key) - .await - .map_err(catalog_store_error)? - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot manifest-list object is missing"))?; - let references = crate::table_catalog::manifest_list_references_from_manifest_list_avro(&manifest_list_object.data) - .map_err(catalog_store_error)?; - if references.is_empty() { - return Err(s3_error!(InvalidRequest, "snapshot manifest-list must reference at least one manifest")); - } - return Ok(references - .into_iter() - .map(|reference| SnapshotManifestLocation { - manifest_path: reference.manifest_path, - sequence_number: reference.sequence_number, - added_snapshot_id: reference.added_snapshot_id, - }) - .collect()); - } - - let Some(manifests) = snapshot.get("manifests").and_then(serde_json::Value::as_array) else { - return Err(s3_error!(InvalidRequest, "snapshot manifest-list is required")); - }; - if manifests.is_empty() { - return Err(s3_error!(InvalidRequest, "snapshot manifests must reference at least one manifest")); - } - manifests - .iter() - .map(|manifest| { - manifest - .as_str() - .filter(|manifest| !manifest.is_empty()) - .map(|manifest| SnapshotManifestLocation { - manifest_path: manifest.to_string(), - sequence_number: None, - added_snapshot_id: None, - }) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot manifest location must be a string")) - }) - .collect() -} - -async fn validate_manifest_data_file_reference( - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &crate::table_catalog::IdentifierSegment, - entry: &crate::table_catalog::TableEntry, - reference: &crate::table_catalog::ManifestDataFileReference, -) -> S3Result<()> -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - table_commit_object_key(bucket, namespace, table, entry, &reference.location, reference.object_kind.clone())?; - let object_key = crate::table_catalog::table_catalog_object_key_from_location(bucket, &reference.location) - .ok_or_else(|| s3_error!(InvalidRequest, "manifest data file location is invalid"))?; - if !metadata_backend - .object_exists(bucket, &object_key) - .await - .map_err(catalog_store_error)? - { - return Err(s3_error!(InvalidRequest, "manifest referenced data file is missing")); - } - Ok(()) -} - -fn table_commit_object_key( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &crate::table_catalog::IdentifierSegment, - entry: &crate::table_catalog::TableEntry, - location: &str, - expected_kind: crate::table_catalog::TableMetadataMaintenanceObjectKind, -) -> S3Result { - let object_key = crate::table_catalog::table_catalog_object_key_from_location(bucket, location) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot object location is invalid"))?; - let warehouse_object_prefix = crate::table_catalog::table_warehouse_object_prefix(entry).map_err(catalog_store_error)?; - let object_kind = - crate::table_catalog::table_maintenance_object_kind(namespace, table, Some(&warehouse_object_prefix), &object_key) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot object is outside the table warehouse"))?; - if object_kind != expected_kind { - return Err(s3_error!(InvalidRequest, "snapshot object kind does not match manifest metadata")); - } - Ok(object_key) -} - -fn apply_set_snapshot_ref_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let ref_name = update - .get("ref-name") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "set-snapshot-ref requires ref-name"))?; - let snapshot_id = update - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "set-snapshot-ref requires snapshot-id"))?; - let reference = update - .as_object() - .ok_or_else(|| s3_error!(InvalidRequest, "set-snapshot-ref must be a JSON object"))? - .iter() - .filter(|(key, _)| key.as_str() != "action" && key.as_str() != "ref-name") - .map(|(key, value)| (key.clone(), value.clone())) - .collect::>(); - ensure_object_field(metadata, "refs")?.insert(ref_name.to_string(), serde_json::Value::Object(reference)); - if ref_name == "main" { - metadata_object_mut(metadata)?.insert("current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id)); - } - Ok(()) -} - -fn apply_remove_snapshots_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let ids = update - .get("snapshot-ids") - .and_then(serde_json::Value::as_array) - .ok_or_else(|| s3_error!(InvalidRequest, "remove-snapshots requires snapshot-ids"))? - .iter() - .filter_map(serde_json::Value::as_i64) - .collect::>(); - ensure_array_field(metadata, "snapshots")?.retain(|snapshot| { - snapshot - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) - }); - ensure_array_field(metadata, "snapshot-log")?.retain(|log| { - log.get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) - }); - Ok(()) -} - -fn apply_remove_snapshot_ref_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let ref_name = update - .get("ref-name") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "remove-snapshot-ref requires ref-name"))?; - ensure_object_field(metadata, "refs")?.remove(ref_name); - Ok(()) -} - -fn apply_set_location_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let location = update - .get("location") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| s3_error!(InvalidRequest, "set-location requires location"))?; - metadata_object_mut(metadata)?.insert("location".to_string(), serde_json::Value::String(location.to_string())); - Ok(()) -} - -fn apply_set_properties_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let updates = update - .get("updates") - .and_then(serde_json::Value::as_object) - .ok_or_else(|| s3_error!(InvalidRequest, "set-properties requires updates"))?; - let properties = ensure_object_field(metadata, "properties")?; - for (key, value) in updates { - let value = value - .as_str() - .ok_or_else(|| s3_error!(InvalidRequest, "table property values must be strings"))?; - properties.insert(key.clone(), serde_json::Value::String(value.to_string())); - } - Ok(()) -} - -fn apply_remove_properties_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { - let removals = update - .get("removals") - .and_then(serde_json::Value::as_array) - .ok_or_else(|| s3_error!(InvalidRequest, "remove-properties requires removals"))?; - let properties = ensure_object_field(metadata, "properties")?; - for removal in removals { - let key = removal - .as_str() - .ok_or_else(|| s3_error!(InvalidRequest, "property removals must be strings"))?; - properties.remove(key); - } - Ok(()) -} - -fn append_previous_metadata_log(metadata: &mut serde_json::Value, previous_metadata_location: &str) -> S3Result<()> { - ensure_array_field(metadata, "metadata-log")?.push(serde_json::json!({ - "timestamp-ms": current_time_millis(), - "metadata-file": previous_metadata_location - })); - Ok(()) -} - -fn metadata_object_mut(metadata: &mut serde_json::Value) -> S3Result<&mut serde_json::Map> { - metadata - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "table metadata must be a JSON object")) -} - -fn ensure_array_field<'a>(metadata: &'a mut serde_json::Value, key: &str) -> S3Result<&'a mut Vec> { - let object = metadata_object_mut(metadata)?; - object - .entry(key.to_string()) - .or_insert_with(|| serde_json::Value::Array(Vec::new())); - object - .get_mut(key) - .and_then(serde_json::Value::as_array_mut) - .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {key} must be an array")) -} - -fn ensure_object_field<'a>( - metadata: &'a mut serde_json::Value, - key: &str, -) -> S3Result<&'a mut serde_json::Map> { - let object = metadata_object_mut(metadata)?; - object - .entry(key.to_string()) - .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new())); - object - .get_mut(key) - .and_then(serde_json::Value::as_object_mut) - .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {key} must be an object")) -} - -fn next_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result { - Ok(last_array_object_i64(metadata, array_key, id_key)?.saturating_add(1)) -} - -fn last_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result { - let values = metadata - .get(array_key) - .and_then(serde_json::Value::as_array) - .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} must be an array"))?; - values - .iter() - .filter_map(|value| value.get(id_key).and_then(serde_json::Value::as_i64)) - .max() - .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} has no {id_key}")) -} - -fn table_commit_operation(metadata: &serde_json::Value) -> String { - metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .and_then(|snapshots| snapshots.last()) - .and_then(|snapshot| snapshot.get("summary")) - .and_then(|summary| summary.get("operation")) - .and_then(serde_json::Value::as_str) - .unwrap_or("commit") - .to_string() -} - -fn namespace_entry_from_create_request( - bucket: &str, - request: CreateNamespaceRequest, -) -> S3Result { - let namespace = namespace_from_segments(&request.namespace)?; - Ok(crate::table_catalog::NamespaceEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - namespace_id: namespace.storage_id(), - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: request.properties, - created_at: None, - updated_at: None, - }) -} - -fn iceberg_rest_error(error_type: &str, status: StatusCode, message: impl Into) -> S3Error { - let mut err = S3Error::with_message(S3ErrorCode::Custom(error_type.into()), message.into()); - err.set_status_code(status); - err -} - -fn catalog_store_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { - match err { - crate::table_catalog::TableCatalogStoreError::NotFound(message) => { - iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_RESOURCE, StatusCode::NOT_FOUND, message) - } - crate::table_catalog::TableCatalogStoreError::Conflict(message) => { - iceberg_rest_error(ICEBERG_ERROR_COMMIT_FAILED, StatusCode::CONFLICT, message) - } - crate::table_catalog::TableCatalogStoreError::Invalid(message) => { - iceberg_rest_error(ICEBERG_ERROR_BAD_REQUEST, StatusCode::BAD_REQUEST, message) - } - crate::table_catalog::TableCatalogStoreError::Internal(message) => { - iceberg_rest_error(ICEBERG_ERROR_REST, StatusCode::INTERNAL_SERVER_ERROR, message) - } - } -} - -fn catalog_store_conflict_error(err: crate::table_catalog::TableCatalogStoreError, conflict_type: &'static str) -> S3Error { - match err { - crate::table_catalog::TableCatalogStoreError::Conflict(message) => { - iceberg_rest_error(conflict_type, StatusCode::CONFLICT, message) - } - err => catalog_store_error(err), - } -} - -fn catalog_store_already_exists_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { - catalog_store_conflict_error(err, ICEBERG_ERROR_ALREADY_EXISTS) -} - -fn catalog_store_namespace_drop_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { - match err { - crate::table_catalog::TableCatalogStoreError::NotFound(message) => { - iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_NAMESPACE, StatusCode::NOT_FOUND, message) - } - err => catalog_store_conflict_error(err, ICEBERG_ERROR_NAMESPACE_NOT_EMPTY), - } -} - -async fn create_namespace_response( - store: &S, - bucket: &str, - request: CreateNamespaceRequest, - table_bucket_enabled: bool, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let entry = namespace_entry_from_create_request(bucket, request)?; - ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; - store - .create_namespace(entry.clone()) - .await - .map_err(catalog_store_already_exists_error)?; - namespace_response_from_entry(entry) -} - -async fn list_namespaces_response(store: &S, bucket: &str, uri: &http::Uri) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let context = RestPageContext { - resource: TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT, - warehouse: bucket, - namespace: None, - }; - let pagination = rest_pagination_from_query(uri, context)?; - let page = match pagination.page_request() { - Some((cursor, limit)) => store - .list_namespaces_page(bucket, cursor, limit) - .await - .map_err(catalog_store_error)?, - None => crate::table_catalog::TableCatalogListPage { - entries: store.list_namespaces(bucket).await.map_err(catalog_store_error)?, - next_cursor: None, - }, - }; - let next_page_token = pagination.next_page_token(page.next_cursor)?; - list_namespaces_response_from_entries(page.entries, next_page_token) -} - -async fn get_namespace_response( - store: &S, - bucket: &str, - namespace: &crate::table_catalog::Namespace, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(entry) = store - .get_namespace(bucket, &namespace.public_name()) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error( - ICEBERG_ERROR_NO_SUCH_NAMESPACE, - StatusCode::NOT_FOUND, - "namespace not found", - )); - }; - namespace_response_from_entry(entry) -} - -async fn namespace_exists_status(store: &S, bucket: &str, namespace: &crate::table_catalog::Namespace) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let exists = store - .get_namespace(bucket, &namespace.public_name()) - .await - .map_err(catalog_store_error)? - .is_some(); - Ok(exists_status(exists)) -} - -async fn drop_namespace_in_store(store: &S, bucket: &str, namespace: &str) -> S3Result<()> -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - store - .drop_namespace(bucket, namespace) - .await - .map_err(catalog_store_namespace_drop_error) -} - -async fn register_table_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - request: RegisterTableRequest, - table_bucket_enabled: bool, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let mut entry = table_entry_from_register_request(bucket, namespace, request)?; - ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &metadata)?; - adopt_registered_metadata_identity(&mut entry, &metadata)?; - store - .register_table(entry.clone()) - .await - .map_err(catalog_store_already_exists_error)?; - Ok(load_table_response_from_entry(entry, metadata)) -} - -async fn create_table_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - request: CreateTableRequest, - table_bucket_enabled: bool, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let (entry, metadata) = table_entry_from_create_table_request(bucket, namespace, request)?; - ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; - let metadata_data = serde_json::to_vec(&metadata) - .map_err(|err| s3_error!(InternalError, "failed to serialize initial table metadata: {}", err))?; - metadata_backend - .put_object( - bucket, - &entry.metadata_location, - metadata_data, - crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, - ) - .await - .map_err(catalog_store_already_exists_error)?; - store - .create_table(entry.clone()) - .await - .map_err(catalog_store_already_exists_error)?; - Ok(load_table_response_from_entry(entry, metadata)) -} - -async fn create_view_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - request: CreateViewRequest, - table_bucket_enabled: bool, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let (entry, metadata) = view_entry_from_create_view_request(bucket, namespace, request)?; - ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; - let metadata_data = serde_json::to_vec(&metadata) - .map_err(|err| s3_error!(InternalError, "failed to serialize initial view metadata: {}", err))?; - metadata_backend - .put_object( - bucket, - &entry.metadata_location, - metadata_data, - crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, - ) - .await - .map_err(catalog_store_already_exists_error)?; - store - .create_view(entry.clone()) - .await - .map_err(catalog_store_already_exists_error)?; - Ok(load_view_response_from_entry(entry, metadata)) -} - -async fn read_table_metadata_json( - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - metadata_location: &str, -) -> S3Result { - let Some(object) = metadata_backend - .read_object(bucket, metadata_location) - .await - .map_err(catalog_store_error)? - else { - return Err(s3_error!(InvalidRequest, "table metadata object not found: {metadata_location}")); - }; - let metadata = serde_json::from_slice::(&object.data) - .map_err(|err| s3_error!(InvalidRequest, "failed to parse table metadata JSON: {}", err))?; - if !metadata.is_object() { - return Err(s3_error!(InvalidRequest, "table metadata JSON must be an object")); - } - Ok(metadata) -} - -async fn list_tables_response( - store: &S, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - uri: &http::Uri, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let namespace = namespace.public_name(); - let context = RestPageContext { - resource: TABLE_CATALOG_TABLE_RESOURCE_ROOT, - warehouse: bucket, - namespace: Some(&namespace), - }; - let pagination = rest_pagination_from_query(uri, context)?; - let page = match pagination.page_request() { - Some((cursor, limit)) => store - .list_tables_page(bucket, &namespace, cursor, limit) - .await - .map_err(catalog_store_error)?, - None => crate::table_catalog::TableCatalogListPage { - entries: store.list_tables(bucket, &namespace).await.map_err(catalog_store_error)?, - next_cursor: None, - }, - }; - let next_page_token = pagination.next_page_token(page.next_cursor)?; - list_tables_response_from_entries(page.entries, next_page_token) -} - -async fn load_table_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(entry) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; - Ok(load_table_response_from_entry(entry, metadata)) -} - -async fn list_views_response( - store: &S, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - uri: &http::Uri, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let namespace = namespace.public_name(); - let context = RestPageContext { - resource: TABLE_CATALOG_VIEW_RESOURCE_ROOT, - warehouse: bucket, - namespace: Some(&namespace), - }; - let pagination = rest_pagination_from_query(uri, context)?; - let page = match pagination.page_request() { - Some((cursor, limit)) => store - .list_views_page(bucket, &namespace, cursor, limit) - .await - .map_err(catalog_store_error)?, - None => crate::table_catalog::TableCatalogListPage { - entries: store.list_views(bucket, &namespace).await.map_err(catalog_store_error)?, - next_cursor: None, - }, - }; - let next_page_token = pagination.next_page_token(page.next_cursor)?; - list_views_response_from_entries(page.entries, next_page_token) -} - -async fn load_view_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - view: &str, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(entry) = store - .load_view(bucket, &namespace.public_name(), view) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_VIEW, StatusCode::NOT_FOUND, "view not found")); - }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; - Ok(load_view_response_from_entry(entry, metadata)) -} - -async fn view_exists_status( - store: &S, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - view: &str, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let exists = store - .load_view(bucket, &namespace.public_name(), view) - .await - .map_err(catalog_store_error)? - .is_some(); - Ok(exists_status(exists)) -} - -async fn replace_view_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - view: &str, - request: RestCommitViewRequest, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(current) = store - .load_view(bucket, &namespace.public_name(), view) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_VIEW, StatusCode::NOT_FOUND, "view not found")); - }; - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_view_commit_requirements(¤t_metadata, &request.requirements)?; - let view_name = crate::table_catalog::IdentifierSegment::parse(view.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid view name: {}", err))?; - let (next_metadata_location, next_metadata) = if let Some(new_metadata_location) = request.new_metadata_location { - if !crate::table_catalog::is_valid_view_metadata_location(namespace, &view_name, &new_metadata_location) { - return Err(s3_error!(InvalidRequest, "metadata location must be inside the view metadata directory")); - } - let target_metadata = read_table_metadata_json(metadata_backend, bucket, &new_metadata_location).await?; - validate_metadata_view_location_in_bucket(bucket, &target_metadata)?; - validate_metadata_matches_current_view_metadata(¤t_metadata, &target_metadata)?; - (new_metadata_location, target_metadata) - } else { - let next_metadata = apply_view_commit_updates(current_metadata.clone(), &request.updates, ¤t.metadata_location)?; - validate_metadata_view_location_in_bucket(bucket, &next_metadata)?; - validate_metadata_matches_current_view_metadata(¤t_metadata, &next_metadata)?; - let (_, metadata_file_token) = standard_commit_ids(request.commit_id); - let next_generation = current.generation.saturating_add(1); - let next_metadata_location = crate::table_catalog::default_view_metadata_file_path( - namespace, - &view_name, - &next_metadata_file_name(next_generation, &metadata_file_token), - ); - let next_metadata_data = serde_json::to_vec(&next_metadata) - .map_err(|err| s3_error!(InternalError, "failed to serialize view metadata update: {}", err))?; - metadata_backend - .put_object( - bucket, - &next_metadata_location, - next_metadata_data, - crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, - ) - .await - .map_err(catalog_store_error)?; - (next_metadata_location, next_metadata) - }; - - let result = store - .replace_view(crate::table_catalog::ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.to_string(), - expected_version_token: request - .expected_version_token - .unwrap_or_else(|| current.version_token.clone()), - expected_metadata_location: request - .expected_metadata_location - .unwrap_or_else(|| current.metadata_location.clone()), - new_metadata_location: next_metadata_location, - }) - .await - .map_err(catalog_store_error)?; - Ok(load_view_response_from_entry(result.view, next_metadata)) -} - -async fn table_exists_status( - store: &S, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let exists = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - .is_some(); - Ok(exists_status(exists)) -} - -async fn load_credentials_response( - store: &S, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - issuer: &dyn TableCredentialIssuer, - principal: Option<&rustfs_credentials::Credentials>, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(entry) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - load_credentials_response_from_entry(&entry, issuer, principal).await -} - -async fn get_table_metadata_location_response( - store: &S, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(entry) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - Ok(table_metadata_location_response_from_entry(entry)) -} - -async fn update_table_metadata_location_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: UpdateTableMetadataLocationRequest, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(current) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; - if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &metadata_location) { - return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); - } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; - let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; - validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; - let commit_request = crate::table_catalog::TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - commit_id: request.commit_id.unwrap_or_else(|| Uuid::new_v4().to_string()), - idempotency_key: request.idempotency_key, - operation: "update-metadata-location".to_string(), - expected_version_token: request.version_token, - expected_metadata_location: current.metadata_location, - new_metadata_location: metadata_location, - requirements: Vec::new(), - writer: Some("rustfs-metadata-location-api".to_string()), - }; - let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; - Ok(table_metadata_location_response_from_entry(result.table)) -} - -async fn commit_table_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: RestCommitTableRequest, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - if request.new_metadata_location.is_none() { - return standard_commit_table_response(store, metadata_backend, bucket, namespace, table, request).await; - } - - let request = table_commit_request_from_rest_request(bucket, namespace, table, request)?; - let Some(current) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &request.new_metadata_location) { - return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); - } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; - let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.new_metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; - validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; - let result = store.commit_table(request).await.map_err(catalog_store_error)?; - Ok(commit_table_response_from_result(result, target_metadata)) -} - -async fn standard_commit_table_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: RestCommitTableRequest, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(current) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_table_commit_requirements(¤t_metadata, &request.requirements)?; - let expected_metadata = current_metadata.clone(); - let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location); - let next_metadata = apply_table_commit_updates(current_metadata, &request.updates, &previous_metadata_location)?; - validate_metadata_table_location_in_bucket(bucket, &next_metadata)?; - validate_metadata_matches_current_metadata(&expected_metadata, &next_metadata)?; - validate_table_snapshot_commit_conflicts( - metadata_backend, - bucket, - namespace, - &table_name, - ¤t, - &expected_metadata, - &request.updates, - ) - .await?; - let (commit_id, metadata_file_token) = standard_commit_ids(request.commit_id); - let next_generation = current.generation.saturating_add(1); - let next_metadata_location = crate::table_catalog::default_table_metadata_file_path( - namespace, - &table_name, - &next_metadata_file_name(next_generation, &metadata_file_token), - ); - let next_metadata_data = serde_json::to_vec(&next_metadata) - .map_err(|err| s3_error!(InternalError, "failed to serialize table metadata update: {}", err))?; - metadata_backend - .put_object( - bucket, - &next_metadata_location, - next_metadata_data, - crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, - ) - .await - .map_err(catalog_store_error)?; - - let commit_request = crate::table_catalog::TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - commit_id, - idempotency_key: request.idempotency_key, - operation: request.operation.unwrap_or_else(|| table_commit_operation(&next_metadata)), - expected_version_token: current.version_token, - expected_metadata_location: current.metadata_location, - new_metadata_location: next_metadata_location, - requirements: request.requirements, - writer: request.writer, - }; - let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; - Ok(commit_table_response_from_result(result, next_metadata)) -} - -async fn drop_table_in_store(store: &S, bucket: &str, namespace: &crate::table_catalog::Namespace, table: &str) -> S3Result<()> -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - store - .drop_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error) -} - -async fn drop_view_in_store(store: &S, bucket: &str, namespace: &crate::table_catalog::Namespace, view: &str) -> S3Result<()> -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - store - .drop_view(bucket, &namespace.public_name(), view) - .await - .map_err(catalog_store_error) -} - -async fn table_metadata_maintenance_response( - store: &crate::table_catalog::ObjectTableCatalogStore, - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: TableMetadataMaintenanceRequest, -) -> S3Result -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - if request.delete && request.commit_snapshot_expiration { - return Err(s3_error!( - InvalidRequest, - "snapshot expiration commit cannot be combined with metadata deletion" - )); - } - if request.delete && request.commit_compaction { - return Err(s3_error!(InvalidRequest, "compaction commit cannot be combined with metadata deletion")); - } - if request.commit_snapshot_expiration && request.commit_compaction { - return Err(s3_error!( - InvalidRequest, - "compaction commit cannot be combined with snapshot expiration commit" - )); - } - if request.commit_compaction && request.compaction.is_none() { - return Err(s3_error!(InvalidRequest, "commit-compaction requires a compaction request")); - } - - let snapshot_expiration_request = request.snapshot_expiration; - let commit_snapshot_expiration = request.commit_snapshot_expiration; - let compaction_request = request.compaction; - let commit_compaction = request.commit_compaction; - let compaction = match compaction_request { - Some(config) if commit_compaction => Some( - store - .commit_table_compaction(bucket, &namespace.public_name(), table, config) - .await - .map_err(catalog_store_error)?, - ), - Some(config) => Some( - store - .plan_table_compaction(bucket, &namespace.public_name(), table, config) - .await - .map_err(catalog_store_error)?, - ), - None => None, - }; - let snapshot_expiration_plan = match snapshot_expiration_request { - Some(config) => Some( - store - .plan_table_snapshot_expiration(bucket, &namespace.public_name(), table, config) - .await - .map_err(catalog_store_error)?, - ), - None => None, - }; - let mut report = store - .run_table_metadata_maintenance_with_retention( - bucket, - &namespace.public_name(), - table, - request.delete, - Some("rustfs-admin".to_string()), - request.retain_recent_metadata_files, - ) - .await - .map_err(catalog_store_error)?; - let snapshot_expiration = match (snapshot_expiration_plan, commit_snapshot_expiration) { - (Some(plan), true) => { - Some(commit_table_snapshot_expiration_response(store, metadata_backend, bucket, namespace, table, plan).await?) - } - (Some(plan), false) => Some(plan), - (None, _) => None, - }; - report.snapshot_expiration = snapshot_expiration; - report.compaction = compaction; - if report.snapshot_expiration.is_some() || report.compaction.is_some() { - let committed_snapshot_expiration = report - .snapshot_expiration - .as_ref() - .is_some_and(|snapshot_expiration| snapshot_expiration.committed_metadata_location.is_some()); - let committed_compaction = report - .compaction - .as_ref() - .is_some_and(|compaction| compaction.committed_metadata_location.is_some()); - match store.put_table_metadata_maintenance_report(&report).await { - Ok(()) => {} - Err(err) if committed_snapshot_expiration || committed_compaction => { - tracing::warn!( - error = %err, - warehouse = bucket, - namespace = namespace.public_name(), - table, - "failed to persist table maintenance report after catalog maintenance commit" - ); - } - Err(err) => return Err(catalog_store_error(err)), - } - } - Ok(report) -} - -async fn commit_table_snapshot_expiration_response( - store: &crate::table_catalog::ObjectTableCatalogStore, - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - mut report: crate::table_catalog::TableSnapshotExpirationReport, -) -> S3Result -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let Some(current) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - if report.table_id != current.table_id || report.current_metadata_location != current.metadata_location { - return Err(s3_error!(PreconditionFailed, "snapshot expiration plan is stale")); - } - if report.manual_review_count > 0 { - return Err(s3_error!(InvalidRequest, "snapshot expiration plan requires manual review before commit")); - } - let expired_snapshot_ids = report - .snapshot_reports - .iter() - .filter(|snapshot| snapshot.state == crate::table_catalog::TableSnapshotExpirationSnapshotState::ExpirationCandidate) - .filter_map(|snapshot| snapshot.snapshot_id) - .collect::>(); - if expired_snapshot_ids.is_empty() { - return Ok(report); - } - - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - let updates = [serde_json::json!({ - "action": "remove-snapshots", - "snapshot-ids": expired_snapshot_ids.clone() - })]; - let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location); - let next_metadata = apply_table_commit_updates(current_metadata.clone(), &updates, &previous_metadata_location)?; - validate_metadata_matches_current_metadata(¤t_metadata, &next_metadata)?; - validate_metadata_table_location_in_bucket(bucket, &next_metadata)?; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - let (commit_id, metadata_file_token) = standard_commit_ids(None); - let next_generation = current.generation.saturating_add(1); - let next_metadata_location = crate::table_catalog::default_table_metadata_file_path( - namespace, - &table_name, - &next_metadata_file_name(next_generation, &metadata_file_token), - ); - let next_metadata_data = serde_json::to_vec(&next_metadata) - .map_err(|err| s3_error!(InternalError, "failed to serialize snapshot expiration metadata: {}", err))?; - metadata_backend - .put_object( - bucket, - &next_metadata_location, - next_metadata_data, - crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, - ) - .await - .map_err(catalog_store_error)?; - - let commit_request = crate::table_catalog::TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - commit_id, - idempotency_key: None, - operation: "expire-snapshots".to_string(), - expected_version_token: current.version_token, - expected_metadata_location: current.metadata_location, - new_metadata_location: next_metadata_location, - requirements: Vec::new(), - writer: Some("rustfs-maintenance".to_string()), - }; - let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; - report.expired_snapshot_ids = expired_snapshot_ids; - report.committed_metadata_location = Some(result.table.metadata_location); - Ok(report) -} - -async fn table_refs_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let Some(entry) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; - let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); - let refs = metadata - .get("refs") - .and_then(serde_json::Value::as_object) - .cloned() - .unwrap_or_default() - .into_iter() - .collect::>(); - let protected_ref_count = refs - .values() - .filter(|reference| { - reference - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_some_and(|snapshot_id| Some(snapshot_id) != current_snapshot_id) - }) - .count(); - let user_defined_ref_count = refs.keys().filter(|name| name.as_str() != "main").count(); - - Ok(TableRefsResponse { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - current_metadata_location: entry.metadata_location, - current_snapshot_id, - protected_ref_count, - user_defined_ref_count, - refs, - }) -} - -async fn put_table_ref_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - ref_name: &str, - request: PutTableRefRequest, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - if !matches!(request.ref_type.as_str(), "branch" | "tag") { - return Err(s3_error!(InvalidRequest, "snapshot ref type must be branch or tag")); - } - let mut update = serde_json::json!({ - "action": "set-snapshot-ref", - "ref-name": ref_name, - "type": request.ref_type, - "snapshot-id": request.snapshot_id - }); - if let Some(value) = request.min_snapshots_to_keep { - update["min-snapshots-to-keep"] = serde_json::Value::from(value); - } - if let Some(value) = request.max_snapshot_age_ms { - update["max-snapshot-age-ms"] = serde_json::Value::from(value); - } - if let Some(value) = request.max_ref_age_ms { - update["max-ref-age-ms"] = serde_json::Value::from(value); - } - let mut requirements = Vec::new(); - if let Some(expected_snapshot_id) = request.expected_snapshot_id { - requirements.push(serde_json::json!({ - "type": "assert-ref-snapshot-id", - "ref": ref_name, - "snapshot-id": expected_snapshot_id - })); - } - standard_commit_table_response( - store, - metadata_backend, - bucket, - namespace, - table, - RestCommitTableRequest { - _identifier: None, - commit_id: request.commit_id, - idempotency_key: request.idempotency_key, - operation: Some("set-snapshot-ref".to_string()), - expected_version_token: None, - expected_metadata_location: None, - new_metadata_location: None, - requirements, - updates: vec![update], - writer: request.writer.or_else(|| Some("rustfs-ref-api".to_string())), - }, - ) - .await -} - -async fn delete_table_ref_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - ref_name: &str, - request: DeleteTableRefRequest, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - if ref_name == "main" { - return Err(s3_error!(InvalidRequest, "main snapshot ref cannot be deleted")); - } - let Some(entry) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; - let reference = metadata - .get("refs") - .and_then(serde_json::Value::as_object) - .and_then(|refs| refs.get(ref_name)); - if reference.is_some_and(snapshot_ref_has_explicit_retention) && !request.force { - return Err(s3_error!(InvalidRequest, "snapshot ref has retention policy; force is required")); - } - let mut requirements = Vec::new(); - if let Some(expected_snapshot_id) = request.expected_snapshot_id { - requirements.push(serde_json::json!({ - "type": "assert-ref-snapshot-id", - "ref": ref_name, - "snapshot-id": expected_snapshot_id - })); - } - standard_commit_table_response( - store, - metadata_backend, - bucket, - namespace, - table, - RestCommitTableRequest { - _identifier: None, - commit_id: request.commit_id, - idempotency_key: request.idempotency_key, - operation: Some("remove-snapshot-ref".to_string()), - expected_version_token: None, - expected_metadata_location: None, - new_metadata_location: None, - requirements, - updates: vec![serde_json::json!({ - "action": "remove-snapshot-ref", - "ref-name": ref_name - })], - writer: request.writer.or_else(|| Some("rustfs-ref-api".to_string())), - }, - ) - .await -} - -fn snapshot_ref_has_explicit_retention(reference: &serde_json::Value) -> bool { - reference.get("min-snapshots-to-keep").is_some() - || reference.get("max-snapshot-age-ms").is_some() - || reference.get("max-ref-age-ms").is_some() -} - -async fn external_catalog_bridge_response( - store: &crate::table_catalog::ObjectTableCatalogStore, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, -) -> S3Result -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let bridge = store - .get_external_catalog_bridge(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)?; - Ok(external_catalog_bridge_response_from_entry(bucket, namespace, table, bridge)) -} - -fn external_catalog_bridge_response_from_entry( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - bridge: Option, -) -> ExternalCatalogBridgeResponse { - let status = if bridge.is_some() { - EXTERNAL_CATALOG_BRIDGE_STATUS_CONFIGURED - } else { - EXTERNAL_CATALOG_BRIDGE_STATUS_UNCONFIGURED - }; - ExternalCatalogBridgeResponse { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - status: status.to_string(), - supported_import: - "register/import an existing Iceberg metadata location into the RustFS catalog; operator-supplied external metadata sync is supported" - .to_string(), - capabilities: external_catalog_bridge_capabilities(), - unsupported_bridges: Vec::new(), - bridge: bridge.map(external_catalog_bridge_state_response), - } -} - -fn external_catalog_bridge_capabilities() -> Vec { - EXTERNAL_CATALOG_BRIDGE_CAPABILITIES - .iter() - .map(|catalog| ExternalCatalogBridgeCapability { - catalog: (*catalog).to_string(), - status: EXTERNAL_CATALOG_BRIDGE_SUPPORTED_STATUS.to_string(), - reason: EXTERNAL_CATALOG_BRIDGE_SUPPORTED_REASON.to_string(), - }) - .collect() -} - -fn external_catalog_bridge_state_response( - entry: crate::table_catalog::ExternalCatalogBridgeEntry, -) -> ExternalCatalogBridgeStateResponse { - ExternalCatalogBridgeStateResponse { - catalog: entry.catalog, - external_catalog_id: entry.external_catalog_id, - external_namespace: entry.external_namespace, - external_table: entry.external_table, - external_table_uuid: entry.external_table_uuid, - metadata_location: entry.metadata_location, - external_version_token: entry.external_version_token, - policy_mode: entry.policy_mode, - credential_mode: entry.credential_mode, - sync_mode: entry.sync_mode, - rollback_strategy: entry.rollback_strategy, - last_sync_status: entry.last_sync_status, - last_synced_metadata_location: entry.last_synced_metadata_location, - properties: entry.properties, - } -} - -fn validate_external_catalog_name(catalog: String) -> S3Result { - let catalog = catalog.trim().to_ascii_lowercase(); - if EXTERNAL_CATALOG_BRIDGE_CAPABILITIES.contains(&catalog.as_str()) { - return Ok(catalog); - } - Err(s3_error!(InvalidRequest, "unsupported external catalog bridge: {catalog}")) -} - -fn validate_external_catalog_field(field_name: &str, value: String) -> S3Result { - let value = value.trim().to_string(); - if value.is_empty() { - return Err(s3_error!(InvalidRequest, "{} must not be empty", field_name)); - } - if value.len() > 512 || value.chars().any(|ch| ch.is_control() || ch == '/' || ch == '\\') { - return Err(s3_error!(InvalidRequest, "{} contains unsupported characters", field_name)); - } - Ok(value) -} - -fn validate_external_catalog_optional_field(field_name: &str, value: Option) -> S3Result> { - value - .map(|value| validate_external_catalog_field(field_name, value)) - .transpose() -} - -fn validate_external_catalog_mode(field_name: &str, value: Option, default_value: &str) -> S3Result { - match value { - Some(value) if value == default_value => Ok(value), - Some(_) => Err(s3_error!( - InvalidRequest, - "{} must be {} until additional external catalog modes are implemented", - field_name, - default_value - )), - None => Ok(default_value.to_string()), - } -} - -fn validate_external_catalog_metadata_location( - namespace: &crate::table_catalog::Namespace, - table: &str, - metadata_location: &str, -) -> S3Result<()> { - let table = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, metadata_location) { - return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); - } - Ok(()) -} - -fn validate_external_catalog_metadata_uuid( - request_table_uuid: Option<&str>, - metadata: &serde_json::Value, -) -> S3Result> { - let metadata_table_uuid = metadata_table_uuid(metadata)?; - if let Some(request_table_uuid) = request_table_uuid - && request_table_uuid != metadata_table_uuid - { - return Err(s3_error!(InvalidRequest, "external table uuid does not match table metadata table-uuid")); - } - Ok(Some(metadata_table_uuid.to_string())) -} - -fn external_catalog_bridge_entry_from_request( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: ExternalCatalogBridgeRequest, -) -> S3Result { - let catalog = validate_external_catalog_name(request.catalog)?; - let external_catalog_id = validate_external_catalog_optional_field("external catalog id", request.external_catalog_id)?; - let external_namespace = validate_external_catalog_field("external namespace", request.external_namespace)?; - let external_table = validate_external_catalog_field("external table", request.external_table)?; - let external_table_uuid = validate_external_catalog_optional_field("external table uuid", request.external_table_uuid)?; - let metadata_location = request - .metadata_location - .map(|metadata_location| table_metadata_location_for_catalog(bucket, &metadata_location)) - .transpose()?; - if let Some(metadata_location) = metadata_location.as_deref() { - validate_external_catalog_metadata_location(namespace, table, metadata_location)?; - } - Ok(crate::table_catalog::ExternalCatalogBridgeEntry { - version: crate::table_catalog::TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - catalog, - external_catalog_id, - external_namespace, - external_table, - external_table_uuid, - metadata_location, - external_version_token: validate_external_catalog_optional_field( - "external version token", - request.external_version_token, - )?, - policy_mode: validate_external_catalog_mode("policy mode", request.policy_mode, EXTERNAL_CATALOG_POLICY_MODE_RUSTFS)?, - credential_mode: validate_external_catalog_mode( - "credential mode", - request.credential_mode, - EXTERNAL_CATALOG_CREDENTIAL_MODE_RUSTFS, - )?, - sync_mode: validate_external_catalog_mode("sync mode", request.sync_mode, EXTERNAL_CATALOG_SYNC_MODE_MANUAL)?, - rollback_strategy: EXTERNAL_CATALOG_ROLLBACK_RETAIN_CURRENT.to_string(), - last_sync_status: None, - last_synced_metadata_location: None, - properties: request.properties, - created_at: None, - updated_at: None, - }) -} - -fn external_catalog_bridge_entry_from_sync_request( - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: &ExternalCatalogBridgeSyncRequest, - external_table_uuid: Option, -) -> S3Result { - let catalog = validate_external_catalog_name(request.catalog.clone())?; - let external_catalog_id = - validate_external_catalog_optional_field("external catalog id", request.external_catalog_id.clone())?; - let external_namespace = validate_external_catalog_field("external namespace", request.external_namespace.clone())?; - let external_table = validate_external_catalog_field("external table", request.external_table.clone())?; - let external_version_token = - validate_external_catalog_optional_field("external version token", request.external_version_token.clone())?; - let rollback_strategy = validate_external_catalog_mode( - "rollback strategy", - request.rollback_strategy.clone(), - EXTERNAL_CATALOG_ROLLBACK_RETAIN_CURRENT, - )?; - Ok(crate::table_catalog::ExternalCatalogBridgeEntry { - version: crate::table_catalog::TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - catalog, - external_catalog_id, - external_namespace, - external_table, - external_table_uuid, - metadata_location: Some(request.metadata_location.clone()), - external_version_token, - policy_mode: validate_external_catalog_mode( - "policy mode", - request.policy_mode.clone(), - EXTERNAL_CATALOG_POLICY_MODE_RUSTFS, - )?, - credential_mode: validate_external_catalog_mode( - "credential mode", - request.credential_mode.clone(), - EXTERNAL_CATALOG_CREDENTIAL_MODE_RUSTFS, - )?, - sync_mode: EXTERNAL_CATALOG_SYNC_MODE_MANUAL.to_string(), - rollback_strategy, - last_sync_status: Some(EXTERNAL_CATALOG_BRIDGE_SYNC_STATUS.to_string()), - last_synced_metadata_location: Some(request.metadata_location.clone()), - properties: request.properties.clone(), - created_at: None, - updated_at: None, - }) -} - -async fn put_external_catalog_bridge_response( - store: &crate::table_catalog::ObjectTableCatalogStore, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: ExternalCatalogBridgeRequest, -) -> S3Result -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let entry = external_catalog_bridge_entry_from_request(bucket, namespace, table, request)?; - let entry = store.put_external_catalog_bridge(entry).await.map_err(catalog_store_error)?; - Ok(external_catalog_bridge_response_from_entry(bucket, namespace, table, Some(entry))) -} - -async fn sync_external_catalog_bridge_response( - store: &crate::table_catalog::ObjectTableCatalogStore, - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: ExternalCatalogBridgeSyncRequest, - table_bucket_enabled: bool, -) -> S3Result -where - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let mut request = request; - request.metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; - request.expected_metadata_location = request - .expected_metadata_location - .map(|metadata_location| table_metadata_location_for_catalog(bucket, &metadata_location)) - .transpose()?; - ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; - validate_external_catalog_metadata_location(namespace, table, &request.metadata_location)?; - let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; - let external_table_uuid = validate_external_catalog_metadata_uuid(request.external_table_uuid.as_deref(), &target_metadata)?; - - let (action, table_response) = if let Some(current) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - { - let expected_version_token = request - .expected_version_token - .clone() - .ok_or_else(|| s3_error!(InvalidRequest, "external catalog sync requires expected-version-token"))?; - let expected_metadata_location = request - .expected_metadata_location - .clone() - .ok_or_else(|| s3_error!(InvalidRequest, "external catalog sync requires expected-metadata-location"))?; - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; - validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; - let result = store - .commit_table(crate::table_catalog::TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - commit_id: request.commit_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string()), - idempotency_key: request.idempotency_key.clone(), - operation: EXTERNAL_CATALOG_SYNC_OPERATION.to_string(), - expected_version_token, - expected_metadata_location, - new_metadata_location: request.metadata_location.clone(), - requirements: Vec::new(), - writer: Some(EXTERNAL_CATALOG_SYNC_WRITER.to_string()), - }) - .await - .map_err(catalog_store_error)?; - ( - EXTERNAL_CATALOG_ACTION_COMMITTED.to_string(), - load_table_response_from_entry(result.table, target_metadata), - ) - } else { - if request.expected_version_token.is_some() || request.expected_metadata_location.is_some() { - return Err(s3_error!( - InvalidRequest, - "external catalog sync cannot use expected table state when registering a missing table" - )); - } - let mut entry = table_entry_from_import_request( - bucket, - namespace, - table, - CatalogImportRequest { - metadata_location: request.metadata_location.clone(), - properties: request.properties.clone(), - }, - )?; - adopt_registered_metadata_identity(&mut entry, &target_metadata)?; - store.register_table(entry.clone()).await.map_err(catalog_store_error)?; - ( - EXTERNAL_CATALOG_ACTION_REGISTERED.to_string(), - load_table_response_from_entry(entry, target_metadata), - ) - }; - - let bridge_entry = external_catalog_bridge_entry_from_sync_request(bucket, namespace, table, &request, external_table_uuid)?; - let bridge_entry = store - .put_external_catalog_bridge(bridge_entry) - .await - .map_err(catalog_store_error)?; - Ok(ExternalCatalogBridgeSyncResponse { - action, - table: table_response, - bridge: external_catalog_bridge_response_from_entry(bucket, namespace, table, Some(bridge_entry)), - }) -} - -async fn catalog_import_response( - store: &S, - metadata_backend: &B, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: CatalogImportRequest, - table_bucket_enabled: bool, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, - B: crate::table_catalog::TableCatalogObjectBackend, -{ - let started = Instant::now(); - let result = async { - ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; - let mut entry = table_entry_from_import_request(bucket, namespace, table, request)?; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &metadata)?; - adopt_registered_metadata_identity(&mut entry, &metadata)?; - if let Some(existing) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - { - if existing.table_uuid == entry.table_uuid - && existing.metadata_location == entry.metadata_location - && existing.warehouse_location == entry.warehouse_location - { - return Ok(load_table_response_from_entry(existing, metadata)); - } - return Err(s3_error!( - PreconditionFailed, - "catalog import target already exists with different table identity or metadata pointer" - )); - } - store.register_table(entry.clone()).await.map_err(catalog_store_error)?; - Ok(load_table_response_from_entry(entry, metadata)) - } - .await; - record_table_catalog_admin_operation_result("import", bucket, &namespace.public_name(), table, started, &result); - result -} - -async fn rollback_table_response( - store: &S, - metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &str, - request: RollbackTableRequest, -) -> S3Result -where - S: crate::table_catalog::TableCatalogStore + ?Sized, -{ - let started = Instant::now(); - let result = async { - let Some(current) = store - .load_table(bucket, &namespace.public_name(), table) - .await - .map_err(catalog_store_error)? - else { - return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); - }; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; - let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; - if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &metadata_location) { - return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); - } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; - let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; - validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; - let commit_request = crate::table_catalog::TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.to_string(), - commit_id: request.commit_id.unwrap_or_else(|| Uuid::new_v4().to_string()), - idempotency_key: request.idempotency_key, - operation: "rollback".to_string(), - expected_version_token: request.version_token, - expected_metadata_location: current.metadata_location, - new_metadata_location: metadata_location, - requirements: Vec::new(), - writer: Some("rustfs-catalog-rollback-api".to_string()), - }; - let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; - Ok(commit_table_response_from_result(result, target_metadata)) - } - .await; - record_table_catalog_admin_operation_result("rollback", bucket, &namespace.public_name(), table, started, &result); - result -} - -pub struct GetCatalogConfigHandler {} - -#[async_trait::async_trait] -impl Operation for GetCatalogConfigHandler { - async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - authorize_table_catalog_request(&req, AdminAction::GetTableCatalogAction).await?; - let warehouse = warehouse_from_config_query(&req.uri)?; - build_json_response(StatusCode::OK, &catalog_config_response(warehouse.as_deref())?) - } -} - -pub struct EnableTableBucketHandler {} - -#[async_trait::async_trait] -impl Operation for EnableTableBucketHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let resource = TableCatalogResource::warehouse(&warehouse); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?; - let store = table_catalog_store()?; - let response = enable_table_bucket_response(&store, &warehouse).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct GetTableBucketHandler {} - -#[async_trait::async_trait] -impl Operation for GetTableBucketHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let resource = TableCatalogResource::warehouse(&warehouse); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableBucketAction).await?; - let store = table_catalog_store()?; - let enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = table_bucket_response(&store, &warehouse, enabled).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct GetTableCatalogMigrationHandler {} - -#[async_trait::async_trait] -impl Operation for GetTableCatalogMigrationHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let resource = TableCatalogResource::warehouse(&warehouse); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCatalogAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_object_store()?; - let started = Instant::now(); - let result = store - .plan_durable_strong_backing_migration(&warehouse) - .await - .map_err(catalog_store_error); - record_table_catalog_admin_operation_result("migration", &warehouse, "", "", started, &result); - let response = result?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct MaterializeTableCatalogMigrationHandler {} - -#[async_trait::async_trait] -impl Operation for MaterializeTableCatalogMigrationHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_object_store()?; - let started = Instant::now(); - let result = store - .materialize_durable_strong_backing_migration(&warehouse) - .await - .map_err(catalog_store_error); - record_table_catalog_admin_operation_result("migration-materialize", &warehouse, "", "", started, &result); - let response = result?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct CancelTableCatalogMigrationHandler {} - -#[async_trait::async_trait] -impl Operation for CancelTableCatalogMigrationHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_object_store()?; - let started = Instant::now(); - let result = store - .cancel_durable_strong_backing_migration(&warehouse) - .await - .map_err(catalog_store_error); - record_table_catalog_admin_operation_result("migration-cancel", &warehouse, "", "", started, &result); - let response = result?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestListNamespacesHandler {} - -#[async_trait::async_trait] -impl Operation for RestListNamespacesHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let resource = TableCatalogResource::warehouse(&warehouse); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let response = list_namespaces_response(&store, &warehouse, &req.uri).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestCreateNamespaceHandler {} - -#[async_trait::async_trait] -impl Operation for RestCreateNamespaceHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let resource = TableCatalogResource::warehouse(&warehouse); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableNamespaceAction).await?; - let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = create_namespace_response(&store, &warehouse, request, table_bucket_enabled).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestGetNamespaceHandler {} - -#[async_trait::async_trait] -impl Operation for RestGetNamespaceHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let response = get_namespace_response(&store, &warehouse, &namespace).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestDropNamespaceHandler {} - -#[async_trait::async_trait] -impl Operation for RestDropNamespaceHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableNamespaceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - drop_namespace_in_store(&store, &warehouse, &namespace.public_name()).await?; - Ok(empty_response(StatusCode::NO_CONTENT)) - } -} - -pub struct RestNamespaceExistsHandler {} - -#[async_trait::async_trait] -impl Operation for RestNamespaceExistsHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - Ok(empty_response(namespace_exists_status(&store, &warehouse, &namespace).await?)) - } -} - -pub struct RestListTablesHandler {} - -#[async_trait::async_trait] -impl Operation for RestListTablesHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let response = list_tables_response(&store, &warehouse, &namespace, &req.uri).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestCreateTableHandler {} - -#[async_trait::async_trait] -impl Operation for RestCreateTableHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CreateTableAction).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = - create_table_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestRegisterTableHandler {} - -#[async_trait::async_trait] -impl Operation for RestRegisterTableHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = - register_table_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestListViewsHandler {} - -#[async_trait::async_trait] -impl Operation for RestListViewsHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let response = list_views_response(&store, &warehouse, &namespace, &req.uri).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestCreateViewHandler {} - -#[async_trait::async_trait] -impl Operation for RestCreateViewHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CreateTableAction).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = - create_view_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestLoadTableHandler {} - -#[async_trait::async_trait] -impl Operation for RestLoadTableHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = load_table_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestTableExistsHandler {} - -#[async_trait::async_trait] -impl Operation for RestTableExistsHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - Ok(empty_response(table_exists_status(&store, &warehouse, &namespace, &table).await?)) - } -} - -pub struct RestLoadCredentialsHandler {} - -#[async_trait::async_trait] -impl Operation for RestLoadCredentialsHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCredentialsAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let principal = table_catalog_request_principal(&req).await?; - let store = table_catalog_store()?; - let issuer = IamTableCredentialIssuer::from_env(); - let response = load_credentials_response(&store, &warehouse, &namespace, &table, &issuer, Some(&principal)).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestCommitTableHandler {} - -#[async_trait::async_trait] -impl Operation for RestCommitTableHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = commit_table_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestDropTableHandler {} - -#[async_trait::async_trait] -impl Operation for RestDropTableHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - drop_table_in_store(&store, &warehouse, &namespace, &table).await?; - Ok(empty_response(StatusCode::NO_CONTENT)) - } -} - -pub struct RestLoadViewHandler {} - -#[async_trait::async_trait] -impl Operation for RestLoadViewHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let view = view_name_from_params(¶ms)?; - let resource = TableCatalogResource::view(&warehouse, &namespace, &view); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = load_view_response(&store, &metadata_backend, &warehouse, &namespace, &view).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestViewExistsHandler {} - -#[async_trait::async_trait] -impl Operation for RestViewExistsHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let view = view_name_from_params(¶ms)?; - let resource = TableCatalogResource::view(&warehouse, &namespace, &view); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - Ok(empty_response(view_exists_status(&store, &warehouse, &namespace, &view).await?)) - } -} - -pub struct RestReplaceViewHandler {} - -#[async_trait::async_trait] -impl Operation for RestReplaceViewHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let view = view_name_from_params(¶ms)?; - let resource = TableCatalogResource::view(&warehouse, &namespace, &view); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = replace_view_response(&store, &metadata_backend, &warehouse, &namespace, &view, request).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestDropViewHandler {} - -#[async_trait::async_trait] -impl Operation for RestDropViewHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let view = view_name_from_params(¶ms)?; - let resource = TableCatalogResource::view(&warehouse, &namespace, &view); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - drop_view_in_store(&store, &warehouse, &namespace, &view).await?; - Ok(empty_response(StatusCode::NO_CONTENT)) - } -} - -pub struct ListTableRefsHandler {} - -#[async_trait::async_trait] -impl Operation for ListTableRefsHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = table_refs_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct PutTableRefHandler {} - -#[async_trait::async_trait] -impl Operation for PutTableRefHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let ref_name = ref_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = - put_table_ref_response(&store, &metadata_backend, &warehouse, &namespace, &table, &ref_name, request).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct DeleteTableRefHandler {} - -#[async_trait::async_trait] -impl Operation for DeleteTableRefHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let ref_name = ref_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body_or_default::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = - delete_table_ref_response(&store, &metadata_backend, &warehouse, &namespace, &table, &ref_name, request).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct GetTableMetadataLocationHandler {} - -#[async_trait::async_trait] -impl Operation for GetTableMetadataLocationHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataLocationAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let response = get_table_metadata_location_response(&store, &warehouse, &namespace, &table).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct UpdateTableMetadataLocationHandler {} - -#[async_trait::async_trait] -impl Operation for UpdateTableMetadataLocationHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = - update_table_metadata_location_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RestTableMetadataMaintenanceHandler {} - -#[async_trait::async_trait] -impl Operation for RestTableMetadataMaintenanceHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_object_store()?; - let response = - table_metadata_maintenance_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct GetTableMaintenanceConfigHandler {} - -#[async_trait::async_trait] -impl Operation for GetTableMaintenanceConfigHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let response = store - .get_table_maintenance_config(&warehouse, &namespace.public_name(), &table) - .await - .map_err(catalog_store_error)?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct PutTableMaintenanceConfigHandler {} - -#[async_trait::async_trait] -impl Operation for PutTableMaintenanceConfigHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableLifecycleAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; - let response = store - .put_table_maintenance_config(&warehouse, &namespace.public_name(), &table, request) - .await - .map_err(catalog_store_error)?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct GetTableMaintenanceJobHandler {} - -#[async_trait::async_trait] -impl Operation for GetTableMaintenanceJobHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let job = job_id_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let Some(response) = store - .get_table_metadata_maintenance_report(&warehouse, &namespace.public_name(), &table, &job) - .await - .map_err(catalog_store_error)? - else { - return Err(s3_error!(InvalidRequest, "maintenance job not found")); - }; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct GetTableMaintenanceSchedulerHandler {} - -#[async_trait::async_trait] -impl Operation for GetTableMaintenanceSchedulerHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let response = store - .get_table_maintenance_scheduler_report(&warehouse, &namespace.public_name(), &table) - .await - .map_err(catalog_store_error)?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RunTableMaintenanceWorkerHandler {} - -#[async_trait::async_trait] -impl Operation for RunTableMaintenanceSchedulerHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body_or_default::(req.input).await?; - let store = table_catalog_store()?; - let response = store - .run_table_maintenance_scheduler_once( - &warehouse, - &namespace.public_name(), - &table, - request.scheduler_id().to_string(), - ) - .await - .map_err(catalog_store_error)?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RunTableMaintenanceSchedulerHandler {} - -#[async_trait::async_trait] -impl Operation for RunTableMaintenanceWorkerHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; - let response = store - .run_table_metadata_maintenance_worker_once( - &warehouse, - &namespace.public_name(), - &table, - request.worker_id().to_string(), - ) - .await - .map_err(catalog_store_error)?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct HeartbeatTableMaintenanceJobHandler {} - -#[async_trait::async_trait] -impl Operation for HeartbeatTableMaintenanceJobHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let job = job_id_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; - let response = store - .heartbeat_table_metadata_maintenance_job( - &warehouse, - &namespace.public_name(), - &table, - &job, - &request.lease_id, - &request.worker_id, - ) - .await - .map_err(catalog_store_error)?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct TableMaintenanceQuarantineHandler {} - -#[async_trait::async_trait] -impl Operation for TableMaintenanceQuarantineHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let job = job_id_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; - let response = store - .apply_table_maintenance_quarantine_operation(&warehouse, &namespace.public_name(), &table, &job, request) - .await - .map_err(catalog_store_error)?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct ExportTableCatalogHandler {} - -#[async_trait::async_trait] -impl Operation for ExportTableCatalogHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let started = Instant::now(); - let result = store - .export_table_catalog_entry(&warehouse, &namespace.public_name(), &table) - .await - .map_err(catalog_store_error); - record_table_catalog_admin_operation_result("export", &warehouse, &namespace.public_name(), &table, started, &result); - let response = result?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct ImportTableCatalogHandler {} - -#[async_trait::async_trait] -impl Operation for ImportTableCatalogHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = - catalog_import_response(&store, &metadata_backend, &warehouse, &namespace, &table, request, table_bucket_enabled) - .await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct ExternalCatalogBridgeHandler {} - -#[async_trait::async_trait] -impl Operation for ExternalCatalogBridgeHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_object_store()?; - let response = external_catalog_bridge_response(&store, &warehouse, &namespace, &table).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct PutExternalCatalogBridgeHandler {} - -#[async_trait::async_trait] -impl Operation for PutExternalCatalogBridgeHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let store = table_catalog_object_store()?; - let response = put_external_catalog_bridge_response(&store, &warehouse, &namespace, &table, request).await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct SyncExternalCatalogBridgeHandler {} - -#[async_trait::async_trait] -impl Operation for SyncExternalCatalogBridgeHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_object_store()?; - if store - .load_table(&warehouse, &namespace.public_name(), &table) - .await - .map_err(catalog_store_error)? - .is_none() - { - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; - } - let request = read_json_body::(req.input).await?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = sync_external_catalog_bridge_response( - &store, - &metadata_backend, - &warehouse, - &namespace, - &table, - request, - table_bucket_enabled, - ) - .await?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct GetTableCatalogDiagnosticsHandler {} - -#[async_trait::async_trait] -impl Operation for GetTableCatalogDiagnosticsHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let config = store - .get_table_maintenance_config(&warehouse, &namespace.public_name(), &table) - .await - .map_err(catalog_store_error)?; - let started = Instant::now(); - let result = store - .diagnose_table_catalog(&warehouse, &namespace.public_name(), &table, config.retain_recent_metadata_files) - .await - .map_err(catalog_store_error); - record_table_catalog_admin_operation_result( - "diagnostics", - &warehouse, - &namespace.public_name(), - &table, - started, - &result, - ); - let response = result?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RecoverTableCatalogHandler {} - -#[async_trait::async_trait] -impl Operation for RecoverTableCatalogHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; - let started = Instant::now(); - let result = store - .recover_table_commits(&warehouse, &namespace.public_name(), &table) - .await - .map_err(catalog_store_error); - record_table_catalog_admin_operation_result("recovery", &warehouse, &namespace.public_name(), &table, started, &result); - let response = result?; - build_json_response(StatusCode::OK, &response) - } -} - -pub struct RollbackTableCatalogHandler {} - -#[async_trait::async_trait] -impl Operation for RollbackTableCatalogHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let warehouse = warehouse_from_params(¶ms)?; - let namespace = namespace_from_params(¶ms)?; - let table = table_name_from_params(¶ms)?; - let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = rollback_table_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; - build_json_response(StatusCode::OK, &response) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::table_catalog::{TableCatalogObjectBackend, TableCatalogStore}; - use std::sync::Arc; - - #[test] - #[serial_test::serial] - fn catalog_config_response_lists_standard_rest_endpoints() { - let response = - temp_env::with_var_unset(crate::table_catalog::ENV_TABLE_CATALOG_BACKING, || catalog_config_response(None)) - .expect("catalog config should build"); - - assert_eq!(response.defaults.get(WAREHOUSE_PROPERTY).map(String::as_str), Some(DEFAULT_WAREHOUSE_ID)); - assert_eq!( - response.defaults.get(CATALOG_ENDPOINT_PREFIX_CONFIG_KEY).map(String::as_str), - Some(TABLE_CATALOG_PREFIX) - ); - assert_eq!( - response - .defaults - .get(CATALOG_COMPAT_ENDPOINT_PREFIX_CONFIG_KEY) - .map(String::as_str), - Some(TABLE_CATALOG_COMPAT_PREFIX) - ); - assert_eq!( - response.defaults.get(CATALOG_BACKING_CONFIG_KEY).map(String::as_str), - Some(crate::table_catalog::TABLE_CATALOG_BACKING_OBJECT) - ); - assert!(!response.defaults.contains_key(PREFIX_PROPERTY)); - assert!(response.overrides.is_empty()); - assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities"); - assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot"); - assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog"); - assert!(response.endpoints.contains(&"GET /v1/{prefix}/namespaces")); - assert!(response.endpoints.contains(&"GET /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"POST /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"DELETE /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"HEAD /v1/{prefix}/namespaces/{namespace}")); - assert!( - response - .endpoints - .contains(&"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials") - ); - assert!(response.endpoints.contains(&"GET /{warehouse}/namespaces")); - assert!(response.endpoints.contains(&"POST /{warehouse}/namespaces")); - assert!(response.endpoints.contains(&"HEAD /{warehouse}/namespaces/{namespace}")); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/register") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables") - ); - assert!(response.endpoints.contains(&"GET /{warehouse}/namespaces/{namespace}/views")); - assert!(response.endpoints.contains(&"POST /{warehouse}/namespaces/{namespace}/views")); - assert!( - response - .endpoints - .contains(&"HEAD /{warehouse}/namespaces/{namespace}/views/{view}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"HEAD /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/credentials") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/views/{view}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/refs") - ); - assert!( - response - .endpoints - .contains(&"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}") - ); - assert!( - response - .endpoints - .contains(&"DELETE /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external") - ); - assert!( - response - .endpoints - .contains(&"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external/sync") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/recovery") - ); - } - - #[test] - #[serial_test::serial] - fn catalog_config_response_reports_durable_strong_backing_override() { - let response = temp_env::with_var( - crate::table_catalog::ENV_TABLE_CATALOG_BACKING, - Some(crate::table_catalog::TABLE_CATALOG_BACKING_DURABLE_STRONG), - || catalog_config_response(None), - ) - .expect("catalog config should build"); - - assert_eq!( - response.overrides.get(CATALOG_BACKING_CONFIG_KEY).map(String::as_str), - Some(crate::table_catalog::TABLE_CATALOG_BACKING_DURABLE_STRONG) - ); - } - - #[test] - #[serial_test::serial] - fn catalog_config_response_uses_requested_warehouse_as_standard_prefix() { - let response = temp_env::with_var_unset(crate::table_catalog::ENV_TABLE_CATALOG_BACKING, || { - catalog_config_response(Some("analytics")) - }) - .expect("catalog config should build"); - - assert_eq!(response.defaults.get(PREFIX_PROPERTY).map(String::as_str), Some("analytics")); - } - - #[test] - fn warehouse_config_query_rejects_empty_and_repeated_values() { - let uri = "/iceberg/v1/config?warehouse=analytics".parse().expect("URI"); - assert_eq!(warehouse_from_config_query(&uri).expect("warehouse query"), Some("analytics".to_string())); - - let uri = "/iceberg/v1/config?warehouse=".parse().expect("URI"); - assert!(warehouse_from_config_query(&uri).is_err()); - - let uri = "/iceberg/v1/config?warehouse=one&warehouse=two".parse().expect("URI"); - assert!(warehouse_from_config_query(&uri).is_err()); - } - - #[test] - fn catalog_conflicts_use_operation_specific_iceberg_errors() { - let already_exists = catalog_store_already_exists_error(crate::table_catalog::TableCatalogStoreError::Conflict( - "table already exists".to_string(), - )); - assert_eq!(already_exists.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_ALREADY_EXISTS.into())); - assert_eq!(already_exists.status_code(), Some(StatusCode::CONFLICT)); - - let namespace_not_empty = catalog_store_namespace_drop_error(crate::table_catalog::TableCatalogStoreError::Conflict( - "namespace is not empty".to_string(), - )); - assert_eq!(namespace_not_empty.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_NAMESPACE_NOT_EMPTY.into())); - assert_eq!(namespace_not_empty.status_code(), Some(StatusCode::CONFLICT)); - - let namespace_not_found = catalog_store_namespace_drop_error(crate::table_catalog::TableCatalogStoreError::NotFound( - "namespace not found".to_string(), - )); - assert_eq!(namespace_not_found.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_NO_SUCH_NAMESPACE.into())); - assert_eq!(namespace_not_found.status_code(), Some(StatusCode::NOT_FOUND)); - } - - #[test] - fn table_catalog_admin_operation_result_labels_are_stable() { - let success: Result<(), ()> = Ok(()); - let failure: Result<(), ()> = Err(()); - - assert_eq!(table_catalog_admin_operation_result_label(&success), "success"); - assert_eq!(table_catalog_admin_operation_result_label(&failure), "failure"); - } - - #[test] - fn table_catalog_handlers_require_table_admin_actions() { - let src = include_str!("table_catalog.rs"); - - assert!( - operation_block(src, "GetCatalogConfigHandler") - .contains("authorize_table_catalog_request(&req, AdminAction::GetTableCatalogAction).await?;") - ); - assert!( - src.contains("validate_admin_request_with_bucket_object("), - "catalog resource auth should pass namespace/table scope into IAM object matching" - ); - - for (handler, action) in [ - ("EnableTableBucketHandler", "AdminAction::SetTableBucketAction"), - ("GetTableBucketHandler", "AdminAction::GetTableBucketAction"), - ("GetTableCatalogMigrationHandler", "AdminAction::GetTableCatalogAction"), - ("RestListNamespacesHandler", "AdminAction::GetTableNamespaceAction"), - ("RestCreateNamespaceHandler", "AdminAction::SetTableNamespaceAction"), - ("RestGetNamespaceHandler", "AdminAction::GetTableNamespaceAction"), - ("RestNamespaceExistsHandler", "AdminAction::GetTableNamespaceAction"), - ("RestDropNamespaceHandler", "AdminAction::DeleteTableNamespaceAction"), - ("RestListTablesHandler", "AdminAction::GetTableAction"), - ("RestCreateTableHandler", "AdminAction::CreateTableAction"), - ("RestRegisterTableHandler", "AdminAction::RegisterTableAction"), - ("RestListViewsHandler", "AdminAction::GetTableMetadataAction"), - ("RestCreateViewHandler", "AdminAction::CreateTableAction"), - ("RestLoadTableHandler", "AdminAction::GetTableMetadataAction"), - ("RestTableExistsHandler", "AdminAction::GetTableAction"), - ("RestLoadCredentialsHandler", "AdminAction::GetTableCredentialsAction"), - ("RestCommitTableHandler", "AdminAction::CommitTableAction"), - ("RestDropTableHandler", "AdminAction::DeleteTableAction"), - ("RestLoadViewHandler", "AdminAction::GetTableMetadataAction"), - ("RestReplaceViewHandler", "AdminAction::CommitTableAction"), - ("RestDropViewHandler", "AdminAction::DeleteTableAction"), - ("ListTableRefsHandler", "AdminAction::GetTableMetadataAction"), - ("GetTableMetadataLocationHandler", "AdminAction::GetTableMetadataLocationAction"), - ("UpdateTableMetadataLocationHandler", "AdminAction::SetTableMetadataLocationAction"), - ("RestTableMetadataMaintenanceHandler", "AdminAction::RunTableMaintenanceAction"), - ("GetTableMaintenanceConfigHandler", "AdminAction::GetTableLifecycleAction"), - ("PutTableMaintenanceConfigHandler", "AdminAction::SetTableLifecycleAction"), - ("GetTableMaintenanceJobHandler", "AdminAction::GetTableLifecycleAction"), - ("GetTableMaintenanceSchedulerHandler", "AdminAction::GetTableLifecycleAction"), - ("RunTableMaintenanceSchedulerHandler", "AdminAction::RunTableMaintenanceAction"), - ("TableMaintenanceQuarantineHandler", "AdminAction::RunTableMaintenanceAction"), - ("ExportTableCatalogHandler", "AdminAction::GetTableMetadataAction"), - ("ImportTableCatalogHandler", "AdminAction::RegisterTableAction"), - ("ExternalCatalogBridgeHandler", "AdminAction::GetTableMetadataAction"), - ("PutExternalCatalogBridgeHandler", "AdminAction::RegisterTableAction"), - ("SyncExternalCatalogBridgeHandler", "AdminAction::SetTableMetadataLocationAction"), - ("GetTableCatalogDiagnosticsHandler", "AdminAction::GetTableMetadataAction"), - ("RecoverTableCatalogHandler", "AdminAction::CommitTableAction"), - ("RollbackTableCatalogHandler", "AdminAction::CommitTableAction"), - ] { - let block = operation_block(src, handler); - assert!( - block.contains(&format!("authorize_table_catalog_resource_request(&req, &resource, {action}).await?;")), - "{handler} should require {action} with catalog resource auth" - ); - assert!( - !block.contains("authorize_table_catalog_request(&req,"), - "{handler} must not use unscoped table catalog authorization" - ); - assert!( - !block.contains("authorize_table_catalog_warehouse_request(&req, &warehouse,"), - "{handler} should not bypass catalog resource auth" - ); - } - - let sync_bridge_block = operation_block(src, "SyncExternalCatalogBridgeHandler"); - assert!( - sync_bridge_block.contains("AdminAction::RegisterTableAction"), - "external catalog sync should require register authorization before creating a missing table" - ); - assert!( - sync_bridge_block.contains(".load_table(&warehouse, &namespace.public_name(), &table)"), - "external catalog sync should branch authorization on current table existence" - ); - - let migration_block = operation_block(src, "GetTableCatalogMigrationHandler"); - assert!( - migration_block.contains("TableCatalogResource::warehouse(&warehouse)"), - "catalog migration dry-run should authorize against the warehouse resource" - ); - for handler in [ - "MaterializeTableCatalogMigrationHandler", - "CancelTableCatalogMigrationHandler", - ] { - let block = operation_block(src, handler); - assert!( - block.contains("authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;"), - "{handler} should require the global catalog migration action" - ); - assert!( - !block.contains("authorize_table_catalog_resource_request("), - "{handler} must not imply that a global backing cutover is warehouse-scoped" - ); - } - - for (handler, action) in [ - ("RestLoadTableHandler", "AdminAction::GetTableMetadataAction"), - ("RestTableExistsHandler", "AdminAction::GetTableAction"), - ("RestLoadCredentialsHandler", "AdminAction::GetTableCredentialsAction"), - ("RestCommitTableHandler", "AdminAction::CommitTableAction"), - ("RestDropTableHandler", "AdminAction::DeleteTableAction"), - ("ListTableRefsHandler", "AdminAction::GetTableMetadataAction"), - ("GetTableMetadataLocationHandler", "AdminAction::GetTableMetadataLocationAction"), - ("UpdateTableMetadataLocationHandler", "AdminAction::SetTableMetadataLocationAction"), - ("RestTableMetadataMaintenanceHandler", "AdminAction::RunTableMaintenanceAction"), - ("GetTableMaintenanceConfigHandler", "AdminAction::GetTableLifecycleAction"), - ("PutTableMaintenanceConfigHandler", "AdminAction::SetTableLifecycleAction"), - ("GetTableMaintenanceJobHandler", "AdminAction::GetTableLifecycleAction"), - ("GetTableMaintenanceSchedulerHandler", "AdminAction::GetTableLifecycleAction"), - ("RunTableMaintenanceSchedulerHandler", "AdminAction::RunTableMaintenanceAction"), - ("TableMaintenanceQuarantineHandler", "AdminAction::RunTableMaintenanceAction"), - ("ExportTableCatalogHandler", "AdminAction::GetTableMetadataAction"), - ("ImportTableCatalogHandler", "AdminAction::RegisterTableAction"), - ("ExternalCatalogBridgeHandler", "AdminAction::GetTableMetadataAction"), - ("PutExternalCatalogBridgeHandler", "AdminAction::RegisterTableAction"), - ("SyncExternalCatalogBridgeHandler", "AdminAction::SetTableMetadataLocationAction"), - ("GetTableCatalogDiagnosticsHandler", "AdminAction::GetTableMetadataAction"), - ("RecoverTableCatalogHandler", "AdminAction::CommitTableAction"), - ("RollbackTableCatalogHandler", "AdminAction::CommitTableAction"), - ] { - let block = operation_block(src, handler); - assert!( - block.contains("TableCatalogResource::table(&warehouse, &namespace, &table)"), - "{handler} should build a table-aware catalog resource" - ); - assert!( - block.contains(&format!("authorize_table_catalog_resource_request(&req, &resource, {action}).await?;")), - "{handler} should authorize against the table-aware catalog resource" - ); - } - } - - #[test] - fn table_catalog_list_handlers_parse_standard_pagination() { - let src = include_str!("table_catalog.rs"); - for (handler, helper_call) in [ - ( - "RestListNamespacesHandler", - "list_namespaces_response(&store, &warehouse, &req.uri).await?", - ), - ( - "RestListTablesHandler", - "list_tables_response(&store, &warehouse, &namespace, &req.uri).await?", - ), - ( - "RestListViewsHandler", - "list_views_response(&store, &warehouse, &namespace, &req.uri).await?", - ), - ] { - let block = operation_block(src, handler); - assert!( - block.contains(helper_call), - "{handler} should pass the request URI to its paginated list helper" - ); - } - } - - #[test] - fn table_catalog_handlers_require_enabled_table_bucket_marker_before_catalog_state() { - let src = include_str!("table_catalog.rs"); - - for handler in [ - "GetTableCatalogMigrationHandler", - "MaterializeTableCatalogMigrationHandler", - "CancelTableCatalogMigrationHandler", - "RestListNamespacesHandler", - "RestCreateNamespaceHandler", - "RestGetNamespaceHandler", - "RestNamespaceExistsHandler", - "RestDropNamespaceHandler", - "RestListTablesHandler", - "RestCreateTableHandler", - "RestRegisterTableHandler", - "RestListViewsHandler", - "RestCreateViewHandler", - "RestLoadTableHandler", - "RestTableExistsHandler", - "RestLoadCredentialsHandler", - "RestCommitTableHandler", - "RestDropTableHandler", - "RestLoadViewHandler", - "RestViewExistsHandler", - "RestReplaceViewHandler", - "RestDropViewHandler", - "ListTableRefsHandler", - "PutTableRefHandler", - "DeleteTableRefHandler", - "GetTableMetadataLocationHandler", - "UpdateTableMetadataLocationHandler", - "RestTableMetadataMaintenanceHandler", - "GetTableMaintenanceConfigHandler", - "PutTableMaintenanceConfigHandler", - "GetTableMaintenanceJobHandler", - "GetTableMaintenanceSchedulerHandler", - "RunTableMaintenanceSchedulerHandler", - "RunTableMaintenanceWorkerHandler", - "HeartbeatTableMaintenanceJobHandler", - "TableMaintenanceQuarantineHandler", - "ExportTableCatalogHandler", - "ImportTableCatalogHandler", - "ExternalCatalogBridgeHandler", - "PutExternalCatalogBridgeHandler", - "SyncExternalCatalogBridgeHandler", - "GetTableCatalogDiagnosticsHandler", - "RecoverTableCatalogHandler", - "RollbackTableCatalogHandler", - ] { - let block = operation_block(src, handler); - assert!( - block.contains("ensure_table_bucket_enabled(&warehouse).await?;") - || block.contains("table_bucket_enabled_from_metadata(&warehouse).await?;"), - "{handler} should require the table bucket metadata marker before catalog state access" - ); - } - } - - #[test] - fn enable_table_bucket_response_writes_metadata_marker_before_catalog_entry() { - let src = include_str!("table_catalog.rs"); - let block = function_block(src, "async fn enable_table_bucket_response"); - let marker_write = block - .find("enable_table_bucket_marker(bucket).await?;") - .expect("enable should write the metadata marker"); - let catalog_entry_write = block - .find("ensure_table_bucket_entry(store, bucket, true).await?;") - .expect("enable should write the catalog entry"); - - assert!( - marker_write < catalog_entry_write, - "enable should write the metadata marker before the catalog entry" - ); - } - - #[test] - fn table_catalog_resource_builds_policy_object_scope() { - let namespace = crate::table_catalog::Namespace::parse("analytics.daily_events").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - - assert_eq!(TableCatalogResource::warehouse("warehouse-a").object_path(), None); - assert_eq!( - TableCatalogResource::namespace("warehouse-a", &namespace) - .object_path() - .as_deref(), - Some("namespaces/analytics/daily_events") - ); - assert_eq!( - TableCatalogResource::table("warehouse-a", &namespace, table.as_str()) - .object_path() - .as_deref(), - Some("namespaces/analytics/daily_events/tables/events") - ); - } - - fn operation_block<'a>(src: &'a str, handler: &str) -> &'a str { - let marker = format!("impl Operation for {handler}"); - let block = src.split_once(&marker).expect("handler impl should exist").1; - let end = block - .find("\npub struct ") - .or_else(|| block.find("\n#[cfg(test)]")) - .unwrap_or(block.len()); - &block[..end] - } - - fn function_block<'a>(src: &'a str, signature: &str) -> &'a str { - let block = src.split_once(signature).expect("function should exist").1; - let end = block - .find("\nfn ") - .or_else(|| block.find("\nasync fn ")) - .unwrap_or(block.len()); - &block[..end] - } - - #[test] - fn rest_catalog_mvp_routes_use_implemented_handlers() { - fn assert_operation() {} - - let _: &EnableTableBucketHandler = &ENABLE_TABLE_BUCKET_HANDLER; - let _: &GetTableBucketHandler = &GET_TABLE_BUCKET_HANDLER; - let _: &GetTableCatalogMigrationHandler = &GET_TABLE_CATALOG_MIGRATION_HANDLER; - let _: &MaterializeTableCatalogMigrationHandler = &MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER; - let _: &CancelTableCatalogMigrationHandler = &CANCEL_TABLE_CATALOG_MIGRATION_HANDLER; - let _: &RestListNamespacesHandler = &LIST_NAMESPACES_HANDLER; - let _: &RestCreateNamespaceHandler = &CREATE_NAMESPACE_HANDLER; - let _: &RestGetNamespaceHandler = &GET_NAMESPACE_HANDLER; - let _: &RestNamespaceExistsHandler = &NAMESPACE_EXISTS_HANDLER; - let _: &RestDropNamespaceHandler = &DROP_NAMESPACE_HANDLER; - let _: &RestListTablesHandler = &LIST_TABLES_HANDLER; - let _: &RestCreateTableHandler = &CREATE_TABLE_HANDLER; - let _: &RestRegisterTableHandler = ®ISTER_TABLE_HANDLER; - let _: &RestListViewsHandler = &LIST_VIEWS_HANDLER; - let _: &RestCreateViewHandler = &CREATE_VIEW_HANDLER; - let _: &RestLoadTableHandler = &LOAD_TABLE_HANDLER; - let _: &RestTableExistsHandler = &TABLE_EXISTS_HANDLER; - let _: &RestLoadCredentialsHandler = &LOAD_CREDENTIALS_HANDLER; - let _: &RestCommitTableHandler = &COMMIT_TABLE_HANDLER; - let _: &RestDropTableHandler = &DROP_TABLE_HANDLER; - let _: &RestLoadViewHandler = &LOAD_VIEW_HANDLER; - let _: &RestReplaceViewHandler = &REPLACE_VIEW_HANDLER; - let _: &RestDropViewHandler = &DROP_VIEW_HANDLER; - let _: &ListTableRefsHandler = &LIST_TABLE_REFS_HANDLER; - let _: &GetTableMetadataLocationHandler = &GET_TABLE_METADATA_LOCATION_HANDLER; - let _: &UpdateTableMetadataLocationHandler = &UPDATE_TABLE_METADATA_LOCATION_HANDLER; - let _: &RestTableMetadataMaintenanceHandler = &TABLE_METADATA_MAINTENANCE_HANDLER; - let _: &GetTableMaintenanceConfigHandler = &GET_TABLE_MAINTENANCE_CONFIG_HANDLER; - let _: &PutTableMaintenanceConfigHandler = &PUT_TABLE_MAINTENANCE_CONFIG_HANDLER; - let _: &GetTableMaintenanceJobHandler = &GET_TABLE_MAINTENANCE_JOB_HANDLER; - let _: &GetTableMaintenanceSchedulerHandler = &GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER; - let _: &RunTableMaintenanceSchedulerHandler = &RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER; - let _: &TableMaintenanceQuarantineHandler = &TABLE_MAINTENANCE_QUARANTINE_HANDLER; - let _: &ExportTableCatalogHandler = &EXPORT_TABLE_CATALOG_HANDLER; - let _: &ImportTableCatalogHandler = &IMPORT_TABLE_CATALOG_HANDLER; - let _: &ExternalCatalogBridgeHandler = &EXTERNAL_CATALOG_BRIDGE_HANDLER; - let _: &PutExternalCatalogBridgeHandler = &PUT_EXTERNAL_CATALOG_BRIDGE_HANDLER; - let _: &SyncExternalCatalogBridgeHandler = &SYNC_EXTERNAL_CATALOG_BRIDGE_HANDLER; - let _: &GetTableCatalogDiagnosticsHandler = &GET_TABLE_CATALOG_DIAGNOSTICS_HANDLER; - let _: &RecoverTableCatalogHandler = &RECOVER_TABLE_CATALOG_HANDLER; - let _: &RollbackTableCatalogHandler = &ROLLBACK_TABLE_CATALOG_HANDLER; - - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - assert_operation::(); - } - - #[test] - fn table_metadata_maintenance_request_uses_conservative_defaults() { - let request: TableMetadataMaintenanceRequest = - serde_json::from_value(serde_json::json!({})).expect("default maintenance request should parse"); - - assert_eq!(request.retain_recent_metadata_files, 0); - assert!(!request.delete); - assert!(request.snapshot_expiration.is_none()); - assert!(!request.commit_snapshot_expiration); - assert!(request.compaction.is_none()); - assert!(!request.commit_compaction); - } - - #[test] - fn table_metadata_maintenance_request_accepts_delete_mode() { - let request: TableMetadataMaintenanceRequest = serde_json::from_value(serde_json::json!({ - "retain-recent-metadata-files": 2, - "delete": true - })) - .expect("metadata maintenance request should parse"); - - assert_eq!(request.retain_recent_metadata_files, 2); - assert!(request.delete); - assert!(request.snapshot_expiration.is_none()); - assert!(!request.commit_snapshot_expiration); - assert!(request.compaction.is_none()); - assert!(!request.commit_compaction); - } - - #[test] - fn table_metadata_maintenance_request_accepts_snapshot_and_compaction_plans() { - let request: TableMetadataMaintenanceRequest = serde_json::from_value(serde_json::json!({ - "commit-snapshot-expiration": true, - "snapshot-expiration": { - "min-snapshots-to-keep": 2, - "max-snapshot-age-ms": 3600000 - }, - "commit-compaction": true, - "compaction": { - "target-file-size-bytes": 536870912, - "small-file-threshold-bytes": 67108864, - "min-input-files": 5, - "max-rewrite-bytes-per-job": 10737418240u64 - } - })) - .expect("metadata maintenance request should parse maintenance planning config"); - - let snapshot_expiration = request - .snapshot_expiration - .expect("snapshot expiration config should be present"); - assert_eq!(snapshot_expiration.min_snapshots_to_keep, 2); - assert_eq!(snapshot_expiration.max_snapshot_age_ms, 3_600_000); - assert!(request.commit_snapshot_expiration); - assert!(request.commit_compaction); - let compaction = request.compaction.expect("compaction config should be present"); - assert_eq!(compaction.target_file_size_bytes, 536_870_912); - assert_eq!(compaction.small_file_threshold_bytes, 67_108_864); - assert_eq!(compaction.min_input_files, 5); - assert_eq!(compaction.max_rewrite_bytes_per_job, 10_737_418_240); - } - - #[test] - fn table_maintenance_scheduler_run_request_uses_stable_default_scheduler_id() { - let request: TableMaintenanceSchedulerRunRequest = - serde_json::from_value(serde_json::json!({})).expect("scheduler run request should parse"); - - assert_eq!(request.scheduler_id(), "rustfs-maintenance-scheduler"); - } - - #[tokio::test] - async fn table_maintenance_scheduler_run_request_empty_body_uses_default_scheduler_id() { - let request: TableMaintenanceSchedulerRunRequest = read_json_body_or_default(Body::empty()) - .await - .expect("bodyless scheduler run should use the default scheduler id"); - - assert_eq!(request.scheduler_id(), "rustfs-maintenance-scheduler"); - } - - #[test] - fn table_maintenance_scheduler_run_request_accepts_scheduler_id() { - let request: TableMaintenanceSchedulerRunRequest = serde_json::from_value(serde_json::json!({ - "scheduler-id": "scheduler-a" - })) - .expect("scheduler run request should parse scheduler id"); - - assert_eq!(request.scheduler_id(), "scheduler-a"); - } - - #[test] - fn table_maintenance_worker_run_request_uses_stable_default_worker_id() { - let request: TableMaintenanceWorkerRunRequest = - serde_json::from_value(serde_json::json!({})).expect("worker run request should parse"); - - assert_eq!(request.worker_id(), "rustfs-maintenance-worker"); - } - - #[test] - fn table_maintenance_worker_run_request_accepts_worker_id() { - let request: TableMaintenanceWorkerRunRequest = serde_json::from_value(serde_json::json!({ - "worker-id": "worker-a" - })) - .expect("worker run request should parse worker id"); - - assert_eq!(request.worker_id(), "worker-a"); - } - - #[test] - fn table_maintenance_heartbeat_request_requires_lease_id() { - let err = serde_json::from_value::(serde_json::json!({ - "worker-id": "worker-a" - })) - .expect_err("heartbeat request should require lease id"); - - assert!(err.to_string().contains("lease-id")); - } - - #[tokio::test] - async fn table_bucket_response_reports_catalog_discovery_without_credentials() { - let store = TestTableCatalogStore::default(); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - - let response = table_bucket_response(&store, "warehouse", true) - .await - .expect("bucket response should build"); - - assert_eq!(response.table_bucket, "warehouse"); - assert!(response.enabled); - assert_eq!(response.catalog_type, crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE); - assert_eq!(response.warehouse_location, "s3://warehouse/"); - assert_eq!(response.catalog_uri, "/iceberg/v1/warehouse"); - assert_eq!(response.compat_catalog_uri, "/_iceberg/v1/warehouse"); - assert_eq!(response.credential_vending, CREDENTIAL_VENDING_UNSUPPORTED); - assert_eq!(response.credential_scope, "warehouse-prefix"); - assert_eq!(response.credential_scope_prefix, "s3://warehouse/"); - assert!(response.catalog_entry_present); - } - - #[test] - fn table_catalog_ingress_requests_reject_unknown_fields() { - assert_rejects_unknown_field::( - "CreateNamespaceRequest", - serde_json::json!({ - "namespace": ["analytics"], - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "RegisterTableRequest", - serde_json::json!({ - "name": "events", - "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "CreateTableRequest", - serde_json::json!({ - "name": "events", - "schema": {}, - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "RestCommitTableRequest", - serde_json::json!({ - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "TableMetadataMaintenanceRequest", - serde_json::json!({ - "delete": true, - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "UpdateTableMetadataLocationRequest", - serde_json::json!({ - "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json", - "version-token": "token-v1", - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "CatalogImportRequest", - serde_json::json!({ - "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "ExternalCatalogBridgeRequest", - serde_json::json!({ - "catalog": "polaris", - "external-namespace": "analytics", - "external-table": "events", - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "ExternalCatalogBridgeSyncRequest", - serde_json::json!({ - "catalog": "glue", - "external-namespace": "analytics", - "external-table": "events", - "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "RollbackTableRequest", - serde_json::json!({ - "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", - "version-token": "token-v2", - "unexpected": true - }), - ); - assert_rejects_unknown_field::( - "TableMaintenanceConfig", - serde_json::json!({ - "version": 1, - "unexpected": true - }), - ); - } - - fn assert_rejects_unknown_field(target: &str, value: serde_json::Value) - where - T: serde::de::DeserializeOwned, - { - let err = match serde_json::from_value::(value) { - Ok(_) => panic!("{target} should reject unknown fields"), - Err(err) => err, - }; - assert!( - err.to_string().contains("unknown field"), - "{target} should reject unknown fields, got: {err}" - ); - } - - #[test] - fn create_namespace_request_uses_rest_namespace_segments_and_properties() { - let request: CreateNamespaceRequest = serde_json::from_value(serde_json::json!({ - "namespace": ["analytics", "daily_events"], - "properties": { - "owner": "lakehouse" - } - })) - .expect("request should parse"); - let namespace = namespace_from_segments(&request.namespace).expect("namespace should be valid"); - let response = namespace_response_from_entry(crate::table_catalog::NamespaceEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "warehouse".to_string(), - namespace: namespace.public_name(), - namespace_id: namespace.storage_id(), - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: request.properties, - created_at: None, - updated_at: None, - }) - .expect("namespace response should build"); - - assert_eq!(namespace.public_name(), "analytics.daily_events"); - assert_eq!(response.namespace, vec!["analytics".to_string(), "daily_events".to_string()]); - assert_eq!(response.properties.get("owner").map(String::as_str), Some("lakehouse")); - } - - #[test] - fn list_tables_response_uses_rest_identifier_shape() { - let namespace = crate::table_catalog::Namespace::parse("analytics.daily_events").expect("namespace should parse"); - let response = list_tables_response_from_entries( - vec![crate::table_catalog::TableEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "warehouse".to_string(), - namespace: namespace.public_name(), - table: "events".to_string(), - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: "s3://warehouse/tables/table-id".to_string(), - metadata_location: - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" - .to_string(), - version_token: "token-v1".to_string(), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }], - None, - ) - .expect("table list response should build"); - - assert_eq!( - response.identifiers[0].namespace, - vec!["analytics".to_string(), "daily_events".to_string()] - ); - assert_eq!(response.identifiers[0].name, "events"); - assert!(response.next_page_token.is_none()); - } - - #[test] - fn rest_pagination_round_trips_context_bound_tokens() { - let context = RestPageContext { - resource: TABLE_CATALOG_TABLE_RESOURCE_ROOT, - warehouse: "warehouse", - namespace: Some("analytics"), - }; - let first_request = "/?pageSize=2".parse::().expect("first page URI should parse"); - let first_pagination = rest_pagination_from_query(&first_request, context).expect("pageSize should start pagination"); - let (cursor, limit) = first_pagination.page_request().expect("pageSize should enable pagination"); - assert_eq!(cursor, None); - assert_eq!(limit.get(), 2); - let next_page_token = first_pagination - .next_page_token(Some("strong:beta".to_string())) - .expect("page token should encode") - .expect("page token should be present"); - let second_request = format!("/?pageSize=2&pageToken={next_page_token}") - .parse::() - .expect("second page URI should parse"); - let second_pagination = rest_pagination_from_query(&second_request, context).expect("continuation token should decode"); - let (cursor, limit) = second_pagination - .page_request() - .expect("continuation should remain paginated"); - assert_eq!(cursor, Some("strong:beta")); - assert_eq!(limit.get(), 2); - assert!( - second_pagination - .next_page_token(None) - .expect("terminal token should build") - .is_none() - ); - - let default_size_request = format!("/?pageToken={next_page_token}") - .parse::() - .expect("continuation URI should parse without pageSize"); - let default_size_pagination = - rest_pagination_from_query(&default_size_request, context).expect("continuation should use the default page size"); - let (cursor, limit) = default_size_pagination - .page_request() - .expect("continuation should remain paginated"); - assert_eq!(cursor, Some("strong:beta")); - assert_eq!(limit.get(), REST_DEFAULT_PAGE_SIZE); - - for other_context in [ - RestPageContext { - resource: TABLE_CATALOG_VIEW_RESOURCE_ROOT, - warehouse: "warehouse", - namespace: Some("analytics"), - }, - RestPageContext { - resource: TABLE_CATALOG_TABLE_RESOURCE_ROOT, - warehouse: "other-warehouse", - namespace: Some("analytics"), - }, - RestPageContext { - resource: TABLE_CATALOG_TABLE_RESOURCE_ROOT, - warehouse: "warehouse", - namespace: Some("other-namespace"), - }, - ] { - let expected_context = rest_page_context_fingerprint(other_context); - let error = decode_rest_page_token(&next_page_token, &expected_context).expect_err("cross-context token should fail"); - assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); - assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); - } - } - - #[test] - fn rest_pagination_rejects_invalid_query_parameters() { - let context = RestPageContext { - resource: TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT, - warehouse: "warehouse", - namespace: None, - }; - for uri in [ - "/?pageSize=0", - "/?pageSize=one", - "/?pageSize=1&pageSize=2", - "/?pageToken=first&pageToken=second", - ] { - let uri = uri.parse::().expect("invalid pagination URI should still parse"); - let error = rest_pagination_from_query(&uri, context).expect_err("invalid pagination query should fail"); - assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into()), "{uri}"); - assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST), "{uri}"); - } - - let oversized_token = "a".repeat(REST_PAGE_TOKEN_MAX_LENGTH + 1); - let oversized_uri = format!("/?pageToken={oversized_token}") - .parse::() - .expect("oversized token URI should parse"); - let error = rest_pagination_from_query(&oversized_uri, context).expect_err("oversized token should fail"); - assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); - assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); - - let empty_token = "/?pageToken=".parse::().expect("empty token URI should parse"); - let pagination = rest_pagination_from_query(&empty_token, context).expect("empty token should start the first page"); - let (cursor, limit) = pagination.page_request().expect("empty token should enable pagination"); - assert_eq!(cursor, None); - assert_eq!(limit.get(), REST_DEFAULT_PAGE_SIZE); - - let capped_size = "/?pageSize=5000" - .parse::() - .expect("large page size URI should parse"); - let pagination = rest_pagination_from_query(&capped_size, context).expect("large page size should be capped"); - let (cursor, limit) = pagination.page_request().expect("pageSize alone should enable pagination"); - assert_eq!(cursor, None); - assert_eq!(limit.get(), REST_MAX_PAGE_SIZE); - - let unpaginated = rest_pagination_from_query(&"/".parse().expect("URI should parse"), context) - .expect("request without pagination should parse"); - assert!(unpaginated.page_request().is_none()); - } - - #[test] - fn rest_pagination_rejects_malformed_token_payloads() { - let context = RestPageContext { - resource: TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT, - warehouse: "warehouse", - namespace: None, - }; - let context_fingerprint = rest_page_context_fingerprint(context); - let encoded_json = |value: serde_json::Value| { - base64_encode_url_safe_no_pad(&serde_json::to_vec(&value).expect("test token should encode")) - }; - let malformed_tokens = [ - "*".to_string(), - base64_encode_url_safe_no_pad(b"not-json"), - encoded_json(serde_json::json!({ - "version": REST_PAGE_TOKEN_VERSION, - "context": context_fingerprint, - "cursor": "strong:alpha", - "unknown": true - })), - encoded_json(serde_json::json!({ - "version": REST_PAGE_TOKEN_VERSION + 1, - "context": context_fingerprint, - "cursor": "strong:alpha" - })), - encoded_json(serde_json::json!({ - "version": REST_PAGE_TOKEN_VERSION, - "context": context_fingerprint, - "cursor": "" - })), - ]; - - for token in malformed_tokens { - let uri = format!( - "/?pageToken={}", - url::form_urlencoded::byte_serialize(token.as_bytes()).collect::() - ) - .parse::() - .expect("malformed token URI should parse"); - let error = rest_pagination_from_query(&uri, context).expect_err("malformed token should fail"); - assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); - assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); - } - } - - #[tokio::test] - async fn rest_list_pagination_covers_namespaces_tables_and_views() { - let store = TestTableCatalogStore::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - for name in ["beta", "alpha"] { - store.namespaces.lock().await.push(crate::table_catalog::NamespaceEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "warehouse".to_string(), - namespace: name.to_string(), - namespace_id: name.to_string(), - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }); - store.tables.lock().await.push(crate::table_catalog::TableEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "warehouse".to_string(), - namespace: namespace.public_name(), - table: name.to_string(), - table_id: format!("table-{name}"), - table_uuid: format!("table-uuid-{name}"), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: format!("s3://warehouse/tables/table-{name}"), - metadata_location: format!("s3://warehouse/tables/table-{name}/metadata/00001.metadata.json"), - version_token: "token-v1".to_string(), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }); - store.views.lock().await.push(crate::table_catalog::ViewEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "warehouse".to_string(), - namespace: namespace.public_name(), - view: name.to_string(), - view_id: format!("view-{name}"), - view_uuid: format!("view-uuid-{name}"), - format: "ICEBERG_VIEW".to_string(), - format_version: 1, - warehouse_location: format!("s3://warehouse/views/view-{name}"), - metadata_location: format!("s3://warehouse/views/view-{name}/metadata/00001.view.json"), - version_token: "token-v1".to_string(), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }); - } - - let first_uri = "/?pageSize=1".parse::().expect("first page URI should parse"); - let namespaces = list_namespaces_response(&store, "warehouse", &first_uri) - .await - .expect("namespace first page should load"); - assert_eq!(namespaces.namespaces, vec![vec!["alpha".to_string()]]); - let namespace_token = namespaces.next_page_token.expect("namespace continuation should exist"); - let namespace_uri = format!("/?pageSize=1&pageToken={namespace_token}") - .parse::() - .expect("namespace continuation URI should parse"); - let namespaces = list_namespaces_response(&store, "warehouse", &namespace_uri) - .await - .expect("namespace second page should load"); - assert_eq!(namespaces.namespaces, vec![vec!["beta".to_string()]]); - assert!(namespaces.next_page_token.is_none()); - - let tables = list_tables_response(&store, "warehouse", &namespace, &first_uri) - .await - .expect("table first page should load"); - assert_eq!(tables.identifiers.len(), 1); - assert_eq!(tables.identifiers[0].name, "alpha"); - let table_token = tables.next_page_token.expect("table continuation should exist"); - let table_uri = format!("/?pageSize=1&pageToken={table_token}") - .parse::() - .expect("table continuation URI should parse"); - let tables = list_tables_response(&store, "warehouse", &namespace, &table_uri) - .await - .expect("table second page should load"); - assert_eq!(tables.identifiers.len(), 1); - assert_eq!(tables.identifiers[0].name, "beta"); - assert!(tables.next_page_token.is_none()); - - let views = list_views_response(&store, "warehouse", &namespace, &first_uri) - .await - .expect("view first page should load"); - assert_eq!(views.identifiers.len(), 1); - assert_eq!(views.identifiers[0].name, "alpha"); - let view_token = views.next_page_token.expect("view continuation should exist"); - let view_uri = format!("/?pageSize=1&pageToken={view_token}") - .parse::() - .expect("view continuation URI should parse"); - let views = list_views_response(&store, "warehouse", &namespace, &view_uri) - .await - .expect("view second page should load"); - assert_eq!(views.identifiers.len(), 1); - assert_eq!(views.identifiers[0].name, "beta"); - assert!(views.next_page_token.is_none()); - - for uri in ["/", "/?pageSize=2"] { - let uri = uri.parse::().expect("list URI should parse"); - let namespaces = list_namespaces_response(&store, "warehouse", &uri) - .await - .expect("namespace exact page should load"); - let tables = list_tables_response(&store, "warehouse", &namespace, &uri) - .await - .expect("table exact page should load"); - let views = list_views_response(&store, "warehouse", &namespace, &uri) - .await - .expect("view exact page should load"); - assert_eq!(namespaces.namespaces.len(), 2); - assert!(namespaces.next_page_token.is_none()); - assert_eq!(tables.identifiers.len(), 2); - assert!(tables.next_page_token.is_none()); - assert_eq!(views.identifiers.len(), 2); - assert!(views.next_page_token.is_none()); - } - } - - #[test] - fn list_responses_expose_null_next_page_token_at_end() { - for value in [ - serde_json::to_value(RestListNamespacesResponse { - namespaces: vec![vec!["analytics".to_string()]], - next_page_token: None, - }) - .expect("namespace response should serialize"), - serde_json::to_value(RestListTablesResponse { - identifiers: Vec::new(), - next_page_token: None, - }) - .expect("table response should serialize"), - serde_json::to_value(RestListViewsResponse { - identifiers: Vec::new(), - next_page_token: None, - }) - .expect("view response should serialize"), - ] { - assert!(value.get("next-page-token").is_some_and(serde_json::Value::is_null)); - } - } - - #[tokio::test] - async fn namespace_exists_status_uses_head_rest_semantics() { - let store = TestTableCatalogStore::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - - assert_eq!( - namespace_exists_status(&store, "warehouse", &namespace) - .await - .expect("missing namespace check should succeed"), - StatusCode::NOT_FOUND - ); - - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - - assert_eq!( - namespace_exists_status(&store, "warehouse", &namespace) - .await - .expect("existing namespace check should succeed"), - StatusCode::NO_CONTENT - ); - } - - #[tokio::test] - async fn table_exists_status_uses_head_rest_semantics() { - let store = TestTableCatalogStore::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - - assert_eq!( - table_exists_status(&store, "warehouse", &namespace, "events") - .await - .expect("missing table check should succeed"), - StatusCode::NOT_FOUND - ); - - store - .create_table(crate::table_catalog::TableEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "warehouse".to_string(), - namespace: namespace.public_name(), - table: "events".to_string(), - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: "s3://warehouse/tables/table-id".to_string(), - metadata_location: - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json" - .to_string(), - version_token: "token-v1".to_string(), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }) - .await - .expect("table should be created"); - - assert_eq!( - table_exists_status(&store, "warehouse", &namespace, "events") - .await - .expect("existing table check should succeed"), - StatusCode::NO_CONTENT - ); - } - - #[test] - fn register_table_request_builds_initial_table_entry() { - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let request: RegisterTableRequest = serde_json::from_value(serde_json::json!({ - "name": "events", - "metadata-location": "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", - "overwrite": false - })) - .expect("request should parse"); - - let entry = table_entry_from_register_request("warehouse", &namespace, request).expect("table entry should build"); - - assert_eq!(entry.table_bucket, "warehouse"); - assert_eq!(entry.namespace, "analytics"); - assert_eq!(entry.table, "events"); - assert_eq!( - entry.metadata_location, - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json" - ); - assert!(entry.properties.is_empty()); - assert_eq!(entry.generation, 1); - assert!(!entry.version_token.is_empty()); - } - - #[test] - fn create_table_request_accepts_standard_iceberg_rest_shape() { - let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ - "name": "events", - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "id", - "required": true, - "type": "long" - } - ] - }, - "partition-spec": { - "spec-id": 0, - "fields": [] - }, - "write-order": { - "order-id": 0, - "fields": [] - }, - "properties": { - "write.format.default": "parquet" - } - })) - .expect("standard create table request should parse"); - - assert_eq!(request.name, "events"); - } - - #[test] - fn commit_table_request_accepts_standard_iceberg_rest_shape() { - let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-table-uuid", - "uuid": "table-uuid" - } - ], - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "lakehouse" - } - } - ] - })) - .expect("standard commit table request should parse"); - - assert_eq!(request.requirements.len(), 1); - } - - #[test] - fn standard_commit_ids_use_uuid_for_metadata_file_when_provided() { - let commit_id = "11111111-1111-4111-8111-111111111111"; - assert_eq!( - standard_commit_ids(Some(commit_id.to_string())), - (commit_id.to_string(), commit_id.to_string()) - ); - } - - #[test] - fn standard_commit_ids_generate_metadata_hash_for_non_uuid_client_id() { - let (commit_id, metadata_file_token) = standard_commit_ids(Some("commit-1".to_string())); - - assert_eq!(commit_id, "commit-1"); - assert_ne!(metadata_file_token, commit_id); - assert_eq!(metadata_file_token, table_catalog_path_hash("commit-1")); - } - - #[tokio::test] - async fn create_table_response_writes_initial_metadata_for_standard_request() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - - let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ - "name": "events", - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "id", - "required": true, - "type": "long" - } - ] - }, - "properties": { - "write.format.default": "parquet" - } - })) - .expect("standard create table request should parse"); - - let response = create_table_response(&store, &metadata_backend, "warehouse", &namespace, request, true) - .await - .expect("table should be created"); - - assert_eq!(response.metadata["format-version"], 2); - assert_eq!(response.metadata["current-schema-id"], 0); - assert_eq!(response.metadata["default-spec-id"], 0); - assert_eq!(response.metadata["default-sort-order-id"], 0); - assert_eq!( - response.metadata["properties"]["write.format.default"], - serde_json::Value::String("parquet".to_string()) - ); - let entry = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - assert_eq!( - response.metadata_location, - format!( - "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001-{}.metadata.json", - entry.table_id - ) - ); - assert_eq!(response.metadata["table-uuid"], entry.table_uuid); - assert!( - metadata_backend - .object_exists("warehouse", &entry.metadata_location) - .await - .expect("metadata object lookup should succeed") - ); - } - - #[tokio::test] - async fn create_table_response_recreates_dropped_identifier_without_overwriting_retained_metadata() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - create_standard_events_table(&store, &metadata_backend, &namespace).await; - let first_entry = store - .load_table("warehouse", "analytics", "events") - .await - .expect("first table lookup should succeed") - .expect("first table should exist"); - let retained_initial = metadata_backend - .read_object("warehouse", &first_entry.metadata_location) - .await - .expect("first metadata lookup should succeed") - .expect("first metadata should exist"); - - let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "first-table" - } - } - ] - })) - .expect("standard commit request should parse"); - standard_commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("first table metadata commit should succeed"); - let first_committed_entry = store - .load_table("warehouse", "analytics", "events") - .await - .expect("committed table lookup should succeed") - .expect("committed table should exist"); - - drop_table_in_store(&store, "warehouse", &namespace, "events") - .await - .expect("first table should drop"); - drop_namespace_in_store(&store, "warehouse", "analytics") - .await - .expect("first namespace should drop"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be recreated"); - - let create_request: CreateTableRequest = serde_json::from_value(serde_json::json!({ - "name": "events", - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "id", - "required": true, - "type": "long" - } - ] - } - })) - .expect("recreate table request should parse"); - let second = create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request, true) - .await - .expect("dropped table identifier should be reusable"); - let second_entry = store - .load_table("warehouse", "analytics", "events") - .await - .expect("recreated table lookup should succeed") - .expect("recreated table should exist"); - - assert_ne!(first_entry.table_id, second_entry.table_id); - assert_ne!(first_entry.table_uuid, second_entry.table_uuid); - assert_ne!(first_entry.metadata_location, second_entry.metadata_location); - assert_ne!(first_committed_entry.metadata_location, second_entry.metadata_location); - assert_eq!(second.metadata["table-uuid"], second_entry.table_uuid); - assert_eq!(second.metadata["metadata-log"], serde_json::json!([])); - assert_eq!( - metadata_backend - .read_object("warehouse", &first_entry.metadata_location) - .await - .expect("retained metadata lookup should succeed") - .expect("retained metadata should still exist") - .data, - retained_initial.data - ); - assert!( - metadata_backend - .object_exists("warehouse", &first_committed_entry.metadata_location) - .await - .expect("committed metadata lookup should succeed") - ); - assert!( - metadata_backend - .object_exists("warehouse", &second_entry.metadata_location) - .await - .expect("recreated metadata lookup should succeed") - ); - } - - #[tokio::test] - async fn concurrent_create_table_responses_keep_one_catalog_winner_with_distinct_metadata() { - let catalog_backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(catalog_backend); - let metadata_backend = TestTableCatalogObjectBackend { - objects: Arc::new(tokio::sync::Mutex::new(BTreeMap::new())), - put_object_barrier: Some(Arc::new(tokio::sync::Barrier::new(2))), - }; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let create_request = || { - serde_json::from_value::(serde_json::json!({ - "name": "events", - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "id", - "required": true, - "type": "long" - } - ] - } - })) - .expect("concurrent create table request should parse") - }; - - let (first, second) = tokio::join!( - create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request(), true,), - create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request(), true,) - ); - assert_ne!(first.is_ok(), second.is_ok(), "exactly one concurrent create should succeed"); - let (winner, loser) = match (first, second) { - (Ok(winner), Err(loser)) | (Err(loser), Ok(winner)) => (winner, loser), - _ => unreachable!("success count was checked above"), - }; - assert!(format!("{loser:?}").contains("AlreadyExistsException")); - - let tables = store - .list_tables("warehouse", "analytics") - .await - .expect("table listing should succeed"); - assert_eq!(tables.len(), 1); - let winner_entry = &tables[0]; - assert_eq!( - winner.metadata_location, - table_metadata_location_for_client("warehouse", &winner_entry.metadata_location) - ); - let metadata_prefix = winner_entry - .metadata_location - .rsplit_once('/') - .map(|(prefix, _)| format!("{prefix}/")) - .expect("metadata location should contain a file name"); - let metadata_objects = metadata_backend - .list_objects("warehouse", &metadata_prefix) - .await - .expect("metadata listing should succeed"); - assert_eq!(metadata_objects.len(), 2); - assert_ne!(metadata_objects[0], metadata_objects[1]); - - let mut table_uuids = Vec::with_capacity(metadata_objects.len()); - for metadata_object in metadata_objects { - let metadata = metadata_backend - .read_object("warehouse", &metadata_object) - .await - .expect("metadata lookup should succeed") - .expect("metadata object should exist"); - let metadata: serde_json::Value = - serde_json::from_slice(&metadata.data).expect("metadata object should contain JSON"); - table_uuids.push( - metadata["table-uuid"] - .as_str() - .expect("metadata should contain table uuid") - .to_string(), - ); - } - table_uuids.sort(); - table_uuids.dedup(); - assert_eq!(table_uuids.len(), 2); - assert!(table_uuids.contains(&winner_entry.table_uuid)); - } - - #[tokio::test] - async fn standard_commit_applies_updates_and_writes_next_metadata() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_uuid = created.metadata["table-uuid"] - .as_str() - .expect("created metadata should have table uuid") - .to_string(); - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; - - let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-table-uuid", - "uuid": table_uuid - }, - { - "type": "assert-current-schema-id", - "current-schema-id": 0 - } - ], - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "lakehouse" - } - }, - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("standard commit table request should parse"); - - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("standard commit should succeed"); - - let metadata_file_prefix = - "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-"; - let metadata_file_suffix = ".metadata.json"; - let generated_commit_id = commit - .metadata_location - .strip_prefix(metadata_file_prefix) - .and_then(|file| file.strip_suffix(metadata_file_suffix)) - .expect("standard commit metadata file should include a UUID suffix"); - Uuid::parse_str(generated_commit_id).expect("metadata file suffix should be a UUID"); - assert_eq!(commit.commit_id, generated_commit_id); - assert_eq!(commit.metadata["properties"]["owner"], serde_json::Value::String("lakehouse".to_string())); - assert_eq!(commit.metadata["current-snapshot-id"], 10); - assert_eq!(commit.metadata["last-sequence-number"], 1); - assert_eq!(commit.metadata["refs"]["main"]["snapshot-id"], 10); - assert_eq!(commit.metadata["metadata-log"][0]["metadata-file"], created.metadata_location); - let committed = store - .load_table("warehouse", "analytics", "events") - .await - .expect("committed table lookup should succeed") - .expect("committed table should exist"); - assert_eq!( - table_metadata_location_for_client("warehouse", &committed.metadata_location), - commit.metadata_location - ); - assert!( - metadata_backend - .object_exists("warehouse", &committed.metadata_location) - .await - .expect("committed metadata lookup should succeed") - ); - } - - #[tokio::test] - async fn standard_commit_uses_client_uuid_commit_id_in_metadata_file_name() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - create_standard_events_table(&store, &metadata_backend, &namespace).await; - - let commit_id = "11111111-1111-4111-8111-111111111111"; - let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "commit-id": commit_id, - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "lakehouse" - } - } - ] - })) - .expect("standard commit table request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("standard commit should succeed"); - - assert_eq!(commit.commit_id, commit_id); - assert_eq!( - commit.metadata_location, - "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-11111111-1111-4111-8111-111111111111.metadata.json" - ); - let committed = store - .load_table("warehouse", "analytics", "events") - .await - .expect("committed table lookup should succeed") - .expect("committed table should exist"); - assert!( - metadata_backend - .object_exists("warehouse", &committed.metadata_location) - .await - .expect("committed metadata lookup should succeed") - ); - } - - #[tokio::test] - async fn standard_commit_accepts_non_uuid_client_commit_id_without_using_it_in_metadata_file_name() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - create_standard_events_table(&store, &metadata_backend, &namespace).await; - - let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "commit-id": "commit-1", - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "lakehouse" - } - } - ] - })) - .expect("standard commit table request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("standard commit should succeed"); - - let metadata_file_prefix = - "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-"; - let metadata_file_suffix = ".metadata.json"; - let metadata_file_token = commit - .metadata_location - .strip_prefix(metadata_file_prefix) - .and_then(|file| file.strip_suffix(metadata_file_suffix)) - .expect("standard commit metadata file should include a safe token suffix"); - assert_eq!(commit.commit_id, "commit-1"); - assert_ne!(metadata_file_token, commit.commit_id); - assert_eq!(metadata_file_token, table_catalog_path_hash("commit-1")); - } - - #[tokio::test] - async fn standard_commit_ignores_generation_only_orphan_metadata_file() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - create_standard_events_table(&store, &metadata_backend, &namespace).await; - metadata_backend - .put_json( - "warehouse", - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json", - serde_json::json!({ - "format-version": 2, - "table-uuid": "orphan", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - let commit_id = "22222222-2222-4222-8222-222222222222"; - let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "commit-id": commit_id, - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "lakehouse" - } - } - ] - })) - .expect("standard commit table request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("standard commit should not collide with generation-only orphan"); - - assert_eq!( - commit.metadata_location, - "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-22222222-2222-4222-8222-222222222222.metadata.json" - ); - assert_eq!(commit.metadata["properties"]["owner"], "lakehouse"); - } - - #[tokio::test] - async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointer_conflict() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - create_standard_events_table(&store, &metadata_backend, &namespace).await; - - let barrier = Arc::new(tokio::sync::Barrier::new(2)); - let metadata_backend = TestTableCatalogObjectBackend { - objects: Arc::clone(&metadata_backend.objects), - put_object_barrier: Some(barrier), - }; - let first_commit_id = "33333333-3333-4333-8333-333333333333"; - let second_commit_id = "44444444-4444-4444-8444-444444444444"; - let first_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "commit-id": first_commit_id, - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "first" - } - } - ] - })) - .expect("first standard commit table request should parse"); - let second_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "commit-id": second_commit_id, - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "second" - } - } - ] - })) - .expect("second standard commit table request should parse"); - - let (first, second) = tokio::join!( - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_request), - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_request) - ); - let success_count = [first.is_ok(), second.is_ok()].into_iter().filter(|ok| *ok).count(); - - assert_eq!(success_count, 1); - assert!( - metadata_backend - .object_exists( - "warehouse", - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-33333333-3333-4333-8333-333333333333.metadata.json" - ) - .await - .expect("first metadata object lookup should succeed") - ); - assert!( - metadata_backend - .object_exists( - "warehouse", - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-44444444-4444-4444-8444-444444444444.metadata.json" - ) - .await - .expect("second metadata object lookup should succeed") - ); - } - - #[tokio::test] - async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_matches() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - let legacy_entry = table_entry_from_register_request( - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: current_location.to_string(), - overwrite: false, - }, - ) - .expect("table entry should build"); - assert_ne!(legacy_entry.table_uuid, "metadata-table-uuid"); - store - .register_table(legacy_entry.clone()) - .await - .expect("legacy table entry should register"); - metadata_backend - .put_json( - "warehouse", - current_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "metadata-table-uuid", - "location": "s3://warehouse/tables/table-id", - "properties": {} - }), - ) - .await; - - let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "set-properties", - "updates": { - "owner": "lakehouse" - } - } - ] - })) - .expect("standard commit table request should parse"); - let committed = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("legacy catalog uuid should not block standard commit"); - - assert_eq!(committed.metadata["table-uuid"], "metadata-table-uuid"); - assert_eq!(committed.metadata["properties"]["owner"], "lakehouse"); - assert_eq!(committed.generation, legacy_entry.generation + 1); - } - - #[tokio::test] - async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_current_metadata() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - let legacy_entry = table_entry_from_register_request( - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: current_location.to_string(), - overwrite: false, - }, - ) - .expect("table entry should build"); - assert_ne!(legacy_entry.table_uuid, "metadata-table-uuid"); - store - .register_table(legacy_entry.clone()) - .await - .expect("legacy table entry should register"); - metadata_backend - .put_json( - "warehouse", - current_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "metadata-table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - let next_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; - metadata_backend - .put_json( - "warehouse", - next_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "metadata-table-uuid", - "location": "s3://warehouse/tables/table-id", - "last-sequence-number": 2 - }), - ) - .await; - - let updated = update_table_metadata_location_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - "events", - UpdateTableMetadataLocationRequest { - metadata_location: next_location.to_string(), - version_token: legacy_entry.version_token, - commit_id: Some("commit-1".to_string()), - idempotency_key: None, - }, - ) - .await - .expect("legacy catalog uuid should not block metadata-location update"); - - assert_eq!(updated.metadata_location, table_metadata_location_for_client("warehouse", next_location)); - assert_eq!(updated.generation, legacy_entry.generation + 1); - } - - #[tokio::test] - async fn table_metadata_maintenance_helper_runs_dry_run_and_delete() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - let old = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; - backend - .put_json_with_mod_time(bucket, &old, serde_json::json!({}), Some(OffsetDateTime::UNIX_EPOCH)) - .await; - backend - .put_json_with_mod_time( - bucket, - ¤t, - serde_json::json!({ - "current-snapshot-id": 20, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "timestamp-ms": 1000, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro" - }, - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-20.avro" - } - ], - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - }), - Some(OffsetDateTime::UNIX_EPOCH), - ) - .await; - - let default_config = store - .get_table_maintenance_config(bucket, "analytics", "events") - .await - .expect("default maintenance config should load"); - assert_eq!(default_config, crate::table_catalog::TableMaintenanceConfig::default()); - let config = store - .put_table_maintenance_config( - bucket, - "analytics", - "events", - crate::table_catalog::TableMaintenanceConfig { - version: crate::table_catalog::TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 2, - delete_enabled: true, - background_enabled: false, - ..Default::default() - }, - ) - .await - .expect("maintenance config should persist"); - assert_eq!(config.retain_recent_metadata_files, 2); - assert!(config.delete_enabled); - let background_config = store - .put_table_maintenance_config( - bucket, - "analytics", - "events", - crate::table_catalog::TableMaintenanceConfig { - version: crate::table_catalog::TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 2, - delete_enabled: true, - background_enabled: true, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - assert!(background_config.background_enabled); - - let dry_run = table_metadata_maintenance_response( - &store, - &backend, - bucket, - &namespace, - "events", - TableMetadataMaintenanceRequest { - retain_recent_metadata_files: 0, - delete: false, - snapshot_expiration: Some(crate::table_catalog::TableSnapshotExpirationConfig { - min_snapshots_to_keep: 1, - max_snapshot_age_ms: 1, - }), - commit_snapshot_expiration: false, - compaction: Some(crate::table_catalog::TableCompactionPlanningConfig { - target_file_size_bytes: 512 * 1024 * 1024, - small_file_threshold_bytes: 64 * 1024 * 1024, - min_input_files: 2, - max_rewrite_bytes_per_job: 1024 * 1024 * 1024, - }), - commit_compaction: false, - }, - ) - .await - .expect("metadata maintenance dry-run should succeed"); - assert_eq!(dry_run.cleanup_candidate_locations, vec![old.clone()]); - assert_eq!(dry_run.deletable_metadata_locations, vec![old.clone()]); - let snapshot_expiration = dry_run - .snapshot_expiration - .as_ref() - .expect("dry-run report should include snapshot expiration planning"); - assert_eq!(snapshot_expiration.expiration_candidate_count, 1); - assert_eq!(snapshot_expiration.current_snapshot_id, Some(20)); - let compaction = dry_run - .compaction - .as_ref() - .expect("dry-run report should include compaction planning"); - assert_eq!( - compaction.status, - crate::table_catalog::TableCompactionPlanningStatus::ManualReviewRequired - ); - assert_eq!(compaction.manual_review_count, 1); - let stored_dry_run = store - .get_table_metadata_maintenance_report(bucket, "analytics", "events", &dry_run.job.job_id) - .await - .expect("maintenance job lookup should succeed") - .expect("maintenance job should be stored"); - assert_eq!(stored_dry_run, dry_run); - assert!( - backend - .object_exists(bucket, &old) - .await - .expect("old metadata lookup should succeed") - ); - - let deleted = table_metadata_maintenance_response( - &store, - &backend, - bucket, - &namespace, - "events", - TableMetadataMaintenanceRequest { - retain_recent_metadata_files: 0, - delete: true, - snapshot_expiration: None, - commit_snapshot_expiration: false, - compaction: None, - commit_compaction: false, - }, - ) - .await - .expect("metadata maintenance delete should succeed"); - assert_eq!(deleted.cleanup_candidate_locations, vec![old.clone()]); - assert_eq!(deleted.deletable_metadata_locations, vec![old.clone()]); - assert!( - !backend - .object_exists(bucket, &old) - .await - .expect("old metadata lookup should succeed after delete") - ); - } - - #[tokio::test] - async fn table_metadata_maintenance_helper_commits_snapshot_expiration() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; - backend - .put_json_with_mod_time( - bucket, - ¤t, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id", - "last-sequence-number": 2, - "last-updated-ms": 2000, - "last-column-id": 1, - "schemas": [], - "current-schema-id": 0, - "partition-specs": [], - "default-spec-id": 0, - "sort-orders": [], - "default-sort-order-id": 0, - "current-snapshot-id": 20, - "metadata-log": [], - "snapshot-log": [ - { - "timestamp-ms": 1000, - "snapshot-id": 10 - }, - { - "timestamp-ms": 2000, - "snapshot-id": 20 - } - ], - "snapshots": [ - { - "snapshot-id": 10, - "timestamp-ms": 1000, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro" - }, - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-20.avro" - } - ], - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - }), - Some(OffsetDateTime::UNIX_EPOCH), - ) - .await; - - let report = table_metadata_maintenance_response( - &store, - &backend, - bucket, - &namespace, - "events", - TableMetadataMaintenanceRequest { - retain_recent_metadata_files: 0, - delete: false, - snapshot_expiration: Some(crate::table_catalog::TableSnapshotExpirationConfig { - min_snapshots_to_keep: 1, - max_snapshot_age_ms: 1, - }), - commit_snapshot_expiration: true, - compaction: None, - commit_compaction: false, - }, - ) - .await - .expect("snapshot expiration commit should succeed"); - - let snapshot_expiration = report - .snapshot_expiration - .as_ref() - .expect("maintenance report should include snapshot expiration"); - assert_eq!(snapshot_expiration.expired_snapshot_ids, vec![10]); - let committed_location = snapshot_expiration - .committed_metadata_location - .as_ref() - .expect("snapshot expiration commit should report committed metadata") - .clone(); - assert_ne!(committed_location, current); - - let entry = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - assert_eq!(entry.metadata_location, committed_location); - assert_eq!(entry.generation, 2); - - let committed_object = backend - .read_object(bucket, &entry.metadata_location) - .await - .expect("committed metadata lookup should succeed") - .expect("committed metadata object should exist"); - let committed_metadata = - serde_json::from_slice::(&committed_object.data).expect("committed metadata should be valid JSON"); - let snapshots = committed_metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .expect("committed metadata should contain snapshots"); - assert_eq!(snapshots.len(), 1); - assert_eq!(snapshots[0].get("snapshot-id").and_then(serde_json::Value::as_i64), Some(20)); - let snapshot_log = committed_metadata - .get("snapshot-log") - .and_then(serde_json::Value::as_array) - .expect("committed metadata should contain snapshot-log"); - assert_eq!(snapshot_log.len(), 1); - assert_eq!( - committed_metadata["metadata-log"][0]["metadata-file"], - serde_json::Value::String(table_metadata_location_for_client(bucket, ¤t)) - ); - assert!( - backend - .object_exists(bucket, ¤t) - .await - .expect("previous metadata lookup should succeed") - ); - } - - #[tokio::test] - async fn table_metadata_maintenance_helper_rejects_snapshot_expiration_manual_review_commit() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; - backend - .put_json_with_mod_time( - bucket, - ¤t, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id", - "current-snapshot-id": 20, - "metadata-log": [], - "snapshot-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "timestamp-ms": 1000, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro" - }, - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-20.avro" - } - ], - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - }, - "audit": { - "snapshot-id": 10, - "type": "tag" - } - } - }), - Some(OffsetDateTime::UNIX_EPOCH), - ) - .await; - - let result = table_metadata_maintenance_response( - &store, - &backend, - bucket, - &namespace, - "events", - TableMetadataMaintenanceRequest { - retain_recent_metadata_files: 0, - delete: false, - snapshot_expiration: Some(crate::table_catalog::TableSnapshotExpirationConfig { - min_snapshots_to_keep: 1, - max_snapshot_age_ms: 1, - }), - commit_snapshot_expiration: true, - compaction: None, - commit_compaction: false, - }, - ) - .await; - - assert!(result.is_err()); - let entry = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - assert_eq!(entry.metadata_location, current); - assert_eq!(entry.generation, 1); - } - - #[tokio::test] - async fn table_metadata_maintenance_helper_rejects_stale_snapshot_expiration_plan() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let next = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - let metadata = serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id", - "current-snapshot-id": 20, - "metadata-log": [], - "snapshot-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "timestamp-ms": 1000, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro" - }, - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-20.avro" - } - ], - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - }); - seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; - backend - .put_json_with_mod_time(bucket, ¤t, metadata.clone(), Some(OffsetDateTime::UNIX_EPOCH)) - .await; - backend - .put_json_with_mod_time(bucket, &next, metadata, Some(OffsetDateTime::UNIX_EPOCH)) - .await; - - let stale_plan = store - .plan_table_snapshot_expiration( - bucket, - "analytics", - "events", - crate::table_catalog::TableSnapshotExpirationConfig { - min_snapshots_to_keep: 1, - max_snapshot_age_ms: 1, - }, - ) - .await - .expect("snapshot expiration plan should build"); - store - .commit_table(crate::table_catalog::TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "advance-pointer".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current, - new_metadata_location: next, - requirements: Vec::new(), - writer: Some("test".to_string()), - }) - .await - .expect("pointer advance should succeed"); - - let result = commit_table_snapshot_expiration_response(&store, &backend, bucket, &namespace, "events", stale_plan).await; - - assert!(result.is_err()); - let entry = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - assert_eq!(entry.generation, 2); - } - - #[tokio::test] - async fn table_metadata_maintenance_helper_rejects_delete_with_snapshot_expiration_commit() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - - let result = table_metadata_maintenance_response( - &store, - &backend, - "warehouse", - &namespace, - "events", - TableMetadataMaintenanceRequest { - retain_recent_metadata_files: 0, - delete: true, - snapshot_expiration: Some(crate::table_catalog::TableSnapshotExpirationConfig { - min_snapshots_to_keep: 1, - max_snapshot_age_ms: 1, - }), - commit_snapshot_expiration: true, - compaction: None, - commit_compaction: false, - }, - ) - .await; - - assert!(result.is_err()); - } - - #[tokio::test] - async fn table_refs_response_reports_current_and_user_defined_refs() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; - backend - .put_json_with_mod_time( - bucket, - ¤t, - serde_json::json!({ - "current-snapshot-id": 10, - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - }, - "audit": { - "snapshot-id": 9, - "type": "tag" - } - } - }), - Some(OffsetDateTime::UNIX_EPOCH), - ) - .await; - - let response = table_refs_response(&store, &backend, bucket, &namespace, "events") - .await - .expect("refs response should load"); - - assert_eq!(response.current_snapshot_id, Some(10)); - assert_eq!(response.protected_ref_count, 1); - assert_eq!(response.user_defined_ref_count, 1); - assert!(response.refs.contains_key("main")); - assert!(response.refs.contains_key("audit")); - } - - #[tokio::test] - async fn external_catalog_bridge_response_lists_supported_operator_bridges() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, bucket, true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - bucket, - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let response = external_catalog_bridge_response(&store, bucket, &namespace, "events") - .await - .expect("bridge response should load"); - - assert_eq!(response.status, "bridge-unconfigured"); - assert!(response.unsupported_bridges.is_empty()); - assert_eq!(response.capabilities.len(), 4); - assert!( - response - .capabilities - .iter() - .any(|bridge| bridge.catalog == "polaris" && bridge.status == "operator-sync-supported") - ); - } - - #[tokio::test] - async fn external_catalog_bridge_persists_identity_and_boundary() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, bucket, true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - bucket, - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - - let configured = put_external_catalog_bridge_response( - &store, - bucket, - &namespace, - "events", - ExternalCatalogBridgeRequest { - catalog: "polaris".to_string(), - external_catalog_id: Some("prod-catalog".to_string()), - external_namespace: "sales.analytics".to_string(), - external_table: "orders".to_string(), - external_table_uuid: Some("table-uuid".to_string()), - metadata_location: None, - external_version_token: Some("polaris-v7".to_string()), - policy_mode: Some("rustfs-authoritative".to_string()), - credential_mode: Some("rustfs-table-credentials".to_string()), - sync_mode: Some("manual".to_string()), - properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), - }, - ) - .await - .expect("bridge should configure"); - - assert_eq!(configured.status, "bridge-configured"); - let bridge = configured.bridge.as_ref().expect("bridge state should be returned"); - assert_eq!(bridge.catalog, "polaris"); - assert_eq!(bridge.external_catalog_id.as_deref(), Some("prod-catalog")); - assert_eq!(bridge.external_namespace, "sales.analytics"); - assert_eq!(bridge.external_table, "orders"); - assert_eq!(bridge.external_version_token.as_deref(), Some("polaris-v7")); - assert_eq!(bridge.policy_mode, "rustfs-authoritative"); - assert_eq!(bridge.credential_mode, "rustfs-table-credentials"); - assert_eq!(bridge.sync_mode, "manual"); - - let loaded = external_catalog_bridge_response(&store, bucket, &namespace, "events") - .await - .expect("bridge response should load"); - assert_eq!(loaded.status, "bridge-configured"); - assert!( - loaded - .capabilities - .iter() - .any(|capability| capability.catalog == "polaris" && capability.status == "operator-sync-supported") - ); - } - - #[tokio::test] - async fn external_catalog_bridge_sync_registers_missing_table_from_snapshot() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - ensure_table_bucket_entry(&store, bucket, true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - bucket, - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let metadata_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - backend - .put_json( - bucket, - &metadata_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - let synced = sync_external_catalog_bridge_response( - &store, - &backend, - bucket, - &namespace, - "events", - ExternalCatalogBridgeSyncRequest { - catalog: "glue".to_string(), - external_catalog_id: Some("aws-glue-prod".to_string()), - external_namespace: "sales".to_string(), - external_table: "orders".to_string(), - external_table_uuid: Some("table-uuid".to_string()), - metadata_location: metadata_location.clone(), - external_version_token: Some("glue-version-1".to_string()), - expected_version_token: None, - expected_metadata_location: None, - commit_id: Some("external-sync-1".to_string()), - idempotency_key: Some("external-sync-idempotency-1".to_string()), - policy_mode: Some("rustfs-authoritative".to_string()), - credential_mode: Some("rustfs-table-credentials".to_string()), - rollback_strategy: Some("retain-current-pointer".to_string()), - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("external sync should register missing table"); - - assert_eq!(synced.action, "registered"); - assert_eq!( - synced.table.metadata_location, - table_metadata_location_for_client(bucket, &metadata_location) - ); - let bridge = synced.bridge.bridge.as_ref().expect("bridge state should be returned"); - assert_eq!(bridge.catalog, "glue"); - assert_eq!(bridge.last_sync_status.as_deref(), Some("synced")); - assert_eq!(bridge.last_synced_metadata_location.as_deref(), Some(metadata_location.as_str())); - assert_eq!(bridge.rollback_strategy, "retain-current-pointer"); - let current = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should be registered"); - assert_eq!(current.metadata_location, metadata_location); - assert_eq!(current.table_uuid, "table-uuid"); - } - - #[tokio::test] - async fn external_catalog_bridge_sync_commits_existing_table_pointer() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let next_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current_location.clone()).await; - backend - .put_json( - bucket, - ¤t_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id", - "last-sequence-number": 1 - }), - ) - .await; - backend - .put_json( - bucket, - &next_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id", - "last-sequence-number": 2 - }), - ) - .await; - - let synced = sync_external_catalog_bridge_response( - &store, - &backend, - bucket, - &namespace, - "events", - ExternalCatalogBridgeSyncRequest { - catalog: "hive-metastore".to_string(), - external_catalog_id: Some("hms-prod".to_string()), - external_namespace: "sales".to_string(), - external_table: "orders".to_string(), - external_table_uuid: Some("table-uuid".to_string()), - metadata_location: table_metadata_location_for_client(bucket, &next_location), - external_version_token: Some("hms-version-2".to_string()), - expected_version_token: Some("token-v1".to_string()), - expected_metadata_location: Some(table_metadata_location_for_client(bucket, ¤t_location)), - commit_id: Some("external-sync-2".to_string()), - idempotency_key: Some("external-sync-idempotency-2".to_string()), - policy_mode: Some("rustfs-authoritative".to_string()), - credential_mode: Some("rustfs-table-credentials".to_string()), - rollback_strategy: Some("retain-current-pointer".to_string()), - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("external sync should commit existing table"); - - assert_eq!(synced.action, "committed"); - assert_eq!(synced.table.metadata_location, table_metadata_location_for_client(bucket, &next_location)); - let current = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - assert_eq!(current.metadata_location, next_location); - assert_eq!(current.generation, 2); - } - - #[tokio::test] - async fn external_catalog_bridge_sync_conflicts_leave_pointer_unchanged() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let next_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current_location.clone()).await; - backend - .put_json( - bucket, - ¤t_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - backend - .put_json( - bucket, - &next_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "different-table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - let result = sync_external_catalog_bridge_response( - &store, - &backend, - bucket, - &namespace, - "events", - ExternalCatalogBridgeSyncRequest { - catalog: "dlf".to_string(), - external_catalog_id: Some("dlf-prod".to_string()), - external_namespace: "sales".to_string(), - external_table: "orders".to_string(), - external_table_uuid: Some("different-table-uuid".to_string()), - metadata_location: next_location.clone(), - external_version_token: Some("dlf-version-2".to_string()), - expected_version_token: Some("token-v1".to_string()), - expected_metadata_location: Some(current_location.clone()), - commit_id: Some("external-sync-3".to_string()), - idempotency_key: Some("external-sync-idempotency-3".to_string()), - policy_mode: Some("rustfs-authoritative".to_string()), - credential_mode: Some("rustfs-table-credentials".to_string()), - rollback_strategy: Some("retain-current-pointer".to_string()), - properties: BTreeMap::new(), - }, - true, - ) - .await; - - assert!(result.is_err()); - let current = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - assert_eq!(current.metadata_location, current_location); - assert_eq!(current.version_token, "token-v1"); - } - - #[test] - fn commit_requirements_reject_mismatched_table_uuid() { - let metadata = serde_json::json!({ - "table-uuid": "actual-table-uuid" - }); - let requirements = vec![serde_json::json!({ - "type": "assert-table-uuid", - "uuid": "stale-table-uuid" - })]; - - assert!(validate_table_commit_requirements(&metadata, &requirements).is_err()); - } - - #[test] - fn snapshot_conflict_requirements_validate_current_snapshot_id() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10 - }); - - let matching = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - })]; - validate_table_commit_requirements(&metadata, &matching).expect("matching current snapshot should pass"); - - let stale = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", - "snapshot-id": 9 - })]; - assert!(validate_table_commit_requirements(&metadata, &stale).is_err()); - - let no_snapshot_metadata = serde_json::json!({}); - let create_like = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", - "snapshot-id": null - })]; - validate_table_commit_requirements(&no_snapshot_metadata, &create_like) - .expect("null current snapshot requirement should pass when no current snapshot exists"); - } - - #[test] - fn snapshot_conflict_rejects_stale_parent_or_sequence_number() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10, - "last-sequence-number": 4, - "snapshots": [ - { - "snapshot-id": 10, - "sequence-number": 4, - "timestamp-ms": 1234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", - "summary": { - "operation": "append" - } - } - ], - "snapshot-log": [], - "metadata-log": [] - }); - - let stale_parent = vec![serde_json::json!({ - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 9, - "sequence-number": 5, - "timestamp-ms": 2234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", - "summary": { - "operation": "append" - } - } - })]; - assert!(apply_table_commit_updates(metadata.clone(), &stale_parent, "metadata/00001.metadata.json").is_err()); - - let stale_sequence = vec![serde_json::json!({ - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 4, - "timestamp-ms": 2234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", - "summary": { - "operation": "append" - } - } - })]; - assert!(apply_table_commit_updates(metadata, &stale_sequence, "metadata/00001.metadata.json").is_err()); - } - - #[test] - fn snapshot_conflict_rejects_unknown_snapshot_operations() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10, - "last-sequence-number": 4, - "snapshots": [ - { - "snapshot-id": 10, - "sequence-number": 4, - "timestamp-ms": 1234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", - "summary": { - "operation": "append" - } - } - ], - "snapshot-log": [], - "metadata-log": [] - }); - - let updates = vec![serde_json::json!({ - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 5, - "timestamp-ms": 2234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", - "summary": { - "operation": "unknown" - } - } - })]; - assert!(apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json").is_err()); - } - - #[tokio::test] - async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let old_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - ¤t_manifest_list, - 10, - 1, - &[(&old_data_file, 0, 1, 10, 1)], - ) - .await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": current_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); - - let overwrite_manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let replacement_data_file = format!("{table_location}/data/part-11.parquet"); - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - &overwrite_manifest_list, - 11, - 2, - &[(&old_data_file, 0, 2, 11, 2), (&replacement_data_file, 0, 1, 11, 2)], - ) - .await; - let overwrite_request_json = serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": overwrite_manifest_list, - "summary": { - "operation": "overwrite" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 11, - "type": "branch" - } - ] - }); - let stale_overwrite_request: RestCommitTableRequest = - serde_json::from_value(overwrite_request_json.clone()).expect("stale overwrite request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", stale_overwrite_request) - .await - .expect_err("deleted file sequence must not change"); - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - &overwrite_manifest_list, - 11, - 2, - &[(&old_data_file, 0, 2, 11, 1), (&replacement_data_file, 0, 1, 11, 2)], - ) - .await; - let overwrite_request: RestCommitTableRequest = - serde_json::from_value(overwrite_request_json).expect("overwrite request should parse"); - - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect("overwrite commit should pass manifest conflict validation"); - - assert_eq!(commit.metadata["current-snapshot-id"], 11); - assert_eq!(commit.metadata["last-sequence-number"], 2); - } - - #[tokio::test] - async fn row_level_conflict_allows_v1_manifest_snapshot() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let manifest = format!("{table_location}/metadata/manifest-10.avro"); - let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, 1)]).await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifests": [ - manifest.clone() - ], - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("append request should parse"); - - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("v1 manifests snapshot should commit"); - - assert_eq!(commit.metadata["current-snapshot-id"], 10); - assert_eq!(commit.metadata["last-sequence-number"], 1); - - let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); - let second_data_file = format!("{table_location}/data/part-11.parquet"); - let second_manifest_list_key = test_snapshot_object_key("warehouse", &second_manifest_list); - metadata_backend - .put_bytes( - "warehouse", - &second_manifest_list_key, - test_manifest_list_avro_entries(&[(&manifest, 1, 10), (&second_manifest, 2, 11)]), - ) - .await; - seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; - let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": second_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 11, - "type": "branch" - } - ] - })) - .expect("second append request should parse"); - - let upgraded = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) - .await - .expect("manifest-list snapshot should inherit a legacy manifest with unknown provenance"); - - assert_eq!(upgraded.metadata["current-snapshot-id"], 11); - assert_eq!(upgraded.metadata["last-sequence-number"], 2); - } - - #[tokio::test] - async fn row_level_conflict_inherits_manifest_list_sequence_numbers() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let manifest = format!("{table_location}/metadata/manifest-10.avro"); - let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 10).await; - seed_test_manifest_with_nullable_sequences(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, None)]) - .await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("append request should parse"); - - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("manifest entry should inherit sequence numbers from manifest list"); - - assert_eq!(commit.metadata["current-snapshot-id"], 10); - assert_eq!(commit.metadata["last-sequence-number"], 1); - } - - #[tokio::test] - async fn row_level_conflict_allows_inherited_manifests_on_append() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let first_manifest = format!("{table_location}/metadata/manifest-10.avro"); - let first_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; - seed_test_manifest(&metadata_backend, "warehouse", &first_manifest, &[(&first_data_file, 0, 1, 10, 1)]).await; - let first_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": first_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("first append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_append) - .await - .expect("first append should commit"); - - let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); - let second_data_file = format!("{table_location}/data/part-11.parquet"); - let second_manifest_list_key = test_snapshot_object_key("warehouse", &second_manifest_list); - metadata_backend - .put_bytes( - "warehouse", - &second_manifest_list_key, - test_manifest_list_avro_entries(&[(&first_manifest, 1, 10), (&second_manifest, 2, 11)]), - ) - .await; - seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; - let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": second_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 11, - "type": "branch" - } - ] - })) - .expect("second append request should parse"); - - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) - .await - .expect("append should preserve inherited manifests"); - - assert_eq!(commit.metadata["current-snapshot-id"], 11); - assert_eq!(commit.metadata["last-sequence-number"], 2); - } - - #[tokio::test] - async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let first_manifest = format!("{table_location}/metadata/manifest-10.avro"); - let first_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; - seed_test_manifest(&metadata_backend, "warehouse", &first_manifest, &[(&first_data_file, 0, 1, 10, 1)]).await; - let first_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": first_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("first append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_append) - .await - .expect("first append should commit"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - - let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); - seed_test_manifest_list(&metadata_backend, "warehouse", &second_manifest_list, &[&first_manifest], 2, 10).await; - let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": second_manifest_list, - "summary": { - "operation": "append" - } - } - } - ] - })) - .expect("second append request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) - .await - .expect_err("inherited manifest identity must not change"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current.metadata_location); - assert_eq!(unchanged.version_token, current.version_token); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn row_level_conflict_rejects_stale_new_manifest_sequence() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let manifest = format!("{table_location}/metadata/manifest-11.avro"); - let data_file = format!("{table_location}/data/part-11.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 11).await; - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 11, 1)]).await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - } - ] - })) - .expect("append request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect_err("new manifest sequence must match the committed snapshot"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current.metadata_location); - assert_eq!(unchanged.version_token, current.version_token); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn row_level_conflict_rejects_stale_added_entry_sequence() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let manifest = format!("{table_location}/metadata/manifest-11.avro"); - let data_file = format!("{table_location}/data/part-11.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 2, 11).await; - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 11, 1)]).await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - } - ] - })) - .expect("append request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect_err("added file sequence must match the new manifest"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current.metadata_location); - assert_eq!(unchanged.version_token, current.version_token); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn row_level_conflict_rejects_historical_change_in_new_manifest() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let manifest = format!("{table_location}/metadata/manifest-11.avro"); - let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 2, 11).await; - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, 1)]).await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - } - ] - })) - .expect("append request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect_err("new manifest must not claim a historical changed entry"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current.metadata_location); - assert_eq!(unchanged.version_token, current.version_token); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn row_level_conflict_allows_add_only_overwrite_snapshot() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let current_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - ¤t_manifest_list, - 10, - 1, - &[(¤t_data_file, 0, 1, 10, 1)], - ) - .await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": current_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); - - let overwrite_manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let added_data_file = format!("{table_location}/data/part-11.parquet"); - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - &overwrite_manifest_list, - 11, - 2, - &[(&added_data_file, 0, 1, 11, 2)], - ) - .await; - let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": overwrite_manifest_list, - "summary": { - "operation": "overwrite" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 11, - "type": "branch" - } - ] - })) - .expect("overwrite request should parse"); - - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect("add-only overwrite should pass conflict validation"); - - assert_eq!(commit.metadata["current-snapshot-id"], 11); - assert_eq!(commit.metadata["last-sequence-number"], 2); - } - - #[tokio::test] - async fn row_level_conflict_rejects_delete_of_non_current_file() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let current_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - ¤t_manifest_list, - 10, - 1, - &[(¤t_data_file, 0, 1, 10, 1)], - ) - .await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": current_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); - let committed = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - - let stale_data_file = format!("{table_location}/data/stale.parquet"); - let stale_key = test_snapshot_object_key("warehouse", &stale_data_file); - metadata_backend.put_bytes("warehouse", &stale_key, b"stale".to_vec()).await; - let overwrite_manifest_list = format!("{table_location}/metadata/snap-11.avro"); - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - &overwrite_manifest_list, - 11, - 2, - &[(&stale_data_file, 0, 2, 11, 2)], - ) - .await; - let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": overwrite_manifest_list, - "summary": { - "operation": "overwrite" - } - } - } - ] - })) - .expect("overwrite request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect_err("stale row-level delete should conflict"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::PreconditionFailed); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, committed.metadata_location); - assert_eq!(unchanged.version_token, committed.version_token); - assert_eq!(unchanged.generation, committed.generation); - } - - #[tokio::test] - async fn row_level_conflict_rejects_append_with_delete_files() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let delete_file = format!("{table_location}/delete/delete-10.parquet"); - seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&delete_file, 1, 1, 10, 1)]).await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - } - ] - })) - .expect("append request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect_err("append must not add delete files"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current.metadata_location); - assert_eq!(unchanged.version_token, current.version_token); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn row_level_conflict_rejects_missing_manifest_before_pointer_update() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let current_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - ¤t_manifest_list, - 10, - 1, - &[(¤t_data_file, 0, 1, 10, 1)], - ) - .await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": current_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); - let committed = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let missing_manifest_list = format!("{table_location}/metadata/missing-snap-11.avro"); - let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": missing_manifest_list, - "summary": { - "operation": "overwrite" - } - } - } - ] - })) - .expect("overwrite request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect_err("missing manifest-list should fail before pointer update"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, committed.metadata_location); - assert_eq!(unchanged.version_token, committed.version_token); - assert_eq!(unchanged.generation, committed.generation); - } - - #[tokio::test] - async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let current_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_snapshot_manifest( - &metadata_backend, - "warehouse", - ¤t_manifest_list, - 10, - 1, - &[(¤t_data_file, 0, 1, 10, 1)], - ) - .await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": current_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); - let committed = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let outside_manifest_list = "s3://warehouse/tables/other-table/metadata/snap-11.avro"; - let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": outside_manifest_list, - "summary": { - "operation": "overwrite" - } - } - } - ] - })) - .expect("overwrite request should parse"); - - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect_err("outside manifest-list should fail before pointer update"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, committed.metadata_location); - assert_eq!(unchanged.version_token, committed.version_token); - assert_eq!(unchanged.generation, committed.generation); - } - - #[tokio::test] - async fn bodyless_ref_delete_uses_default_request_options() { - let request: DeleteTableRefRequest = read_json_body_or_default(Body::empty()) - .await - .expect("bodyless ref delete should use default request options"); - - assert!(request.expected_snapshot_id.is_none()); - assert!(!request.force); - assert!(request.commit_id.is_none()); - assert!(request.idempotency_key.is_none()); - assert!(request.writer.is_none()); - } - - #[test] - fn table_updates_reject_unknown_actions() { - let metadata = serde_json::json!({ - "metadata-log": [] - }); - let updates = vec![serde_json::json!({ - "action": "rewrite-everything" - })]; - - assert!(apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json").is_err()); - } - - #[test] - fn table_location_updates_must_stay_inside_bucket() { - let metadata = serde_json::json!({ - "location": "s3://warehouse/tables/table-id", - "metadata-log": [] - }); - let updates = vec![serde_json::json!({ - "action": "set-location", - "location": "s3://other-warehouse/tables/table-id" - })]; - - let updated = apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json") - .expect("set-location should update metadata before boundary validation"); - - assert!(validate_metadata_table_location_in_bucket("warehouse", &updated).is_err()); - } - - #[test] - fn create_view_request_accepts_standard_iceberg_rest_shape() { - let request: CreateViewRequest = serde_json::from_value(serde_json::json!({ - "name": "recent_events", - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "id", - "required": true, - "type": "long" - } - ] - }, - "view-version": { - "version-id": 1, - "schema-id": 0, - "summary": { - "engine-name": "spark", - "engine-version": "3.5.0" - }, - "default-catalog": "warehouse", - "default-namespace": ["analytics"], - "representations": [ - { - "type": "sql", - "sql": "SELECT id FROM analytics.events WHERE ts >= current_date()", - "dialect": "spark" - } - ] - }, - "properties": { - "comment": "recent event ids" - } - })) - .expect("standard create view request should parse"); - - assert_eq!(request.name, "recent_events"); - assert_eq!(request.properties.get("comment").map(String::as_str), Some("recent event ids")); - } - - #[test] - fn create_view_request_accepts_deep_warehouse_location() { - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let deep_prefix = (0..80).map(|index| format!("level-{index}")).collect::>().join("/"); - let location = format!("s3://warehouse/{deep_prefix}"); - assert!(crate::table_catalog::validate_table_warehouse_location("warehouse", &location).is_err()); - let request: CreateViewRequest = serde_json::from_value(serde_json::json!({ - "name": "recent_events", - "location": location, - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [] - }, - "view-version": { - "version-id": 1, - "schema-id": 0, - "summary": { - "engine-name": "spark" - }, - "default-catalog": "warehouse", - "default-namespace": ["analytics"], - "representations": [ - { - "type": "sql", - "sql": "SELECT 1", - "dialect": "spark" - } - ] - } - })) - .expect("deep create view request should parse"); - - let (entry, metadata) = view_entry_from_create_view_request("warehouse", &namespace, request) - .expect("view location should not inherit table warehouse index depth limits"); - - assert_eq!(entry.warehouse_location, format!("s3://warehouse/{deep_prefix}")); - validate_metadata_view_location_in_bucket("warehouse", &metadata) - .expect("view metadata location should not inherit table warehouse index depth limits"); - } - - #[tokio::test] - async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - - let create_request: CreateViewRequest = serde_json::from_value(serde_json::json!({ - "name": "recent_events", - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "id", - "required": true, - "type": "long" - } - ] - }, - "view-version": { - "version-id": 1, - "schema-id": 0, - "summary": { - "engine-name": "spark" - }, - "default-catalog": "warehouse", - "default-namespace": ["analytics"], - "representations": [ - { - "type": "sql", - "sql": "SELECT id FROM analytics.events", - "dialect": "spark" - } - ] - } - })) - .expect("standard create view request should parse"); - - let created = create_view_response(&store, &metadata_backend, "warehouse", &namespace, create_request, true) - .await - .expect("view should be created"); - assert_eq!(created.metadata["format-version"], 1); - assert_eq!(created.metadata["current-version-id"], 1); - assert_eq!(created.metadata["versions"][0]["representations"][0]["dialect"], "spark"); - assert!( - metadata_backend - .object_exists("warehouse", &created.metadata_location) - .await - .expect("view metadata object lookup should succeed") - ); - - let unpaginated_uri = "/".parse::().expect("list URI should parse"); - let listed = list_views_response(&store, "warehouse", &namespace, &unpaginated_uri) - .await - .expect("views should list"); - assert_eq!(listed.identifiers.len(), 1); - assert_eq!(listed.identifiers[0].name, "recent_events"); - - let loaded = load_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events") - .await - .expect("view should load"); - assert_eq!(loaded.metadata_location, created.metadata_location); - let replace_request: RestCommitViewRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-view-version", - "view-version": { - "version-id": 2, - "schema-id": 0, - "summary": { - "engine-name": "spark" - }, - "default-catalog": "warehouse", - "default-namespace": ["analytics"], - "representations": [ - { - "type": "sql", - "sql": "SELECT id FROM analytics.events WHERE id > 10", - "dialect": "spark" - } - ] - } - }, - { - "action": "set-current-view-version", - "view-version-id": 2 - } - ] - })) - .expect("replace view request should parse"); - let replaced = - replace_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events", replace_request) - .await - .expect("view should replace"); - assert_ne!(replaced.metadata_location, created.metadata_location); - assert_eq!(replaced.metadata["current-version-id"], 2); - assert_eq!( - replaced.metadata["version-log"] - .as_array() - .expect("version log should be an array") - .len(), - 2 - ); - - drop_view_in_store(&store, "warehouse", &namespace, "recent_events") - .await - .expect("view should drop"); - let listed = list_views_response(&store, "warehouse", &namespace, &unpaginated_uri) - .await - .expect("views should list after drop"); - assert!(listed.identifiers.is_empty()); - - let recreate_request: CreateViewRequest = serde_json::from_value(serde_json::json!({ - "name": "recent_events", - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "id", - "required": true, - "type": "long" - } - ] - }, - "view-version": { - "version-id": 1, - "schema-id": 0, - "summary": { - "engine-name": "spark" - }, - "default-catalog": "warehouse", - "default-namespace": ["analytics"], - "representations": [ - { - "type": "sql", - "sql": "SELECT id FROM analytics.events WHERE id > 100", - "dialect": "spark" - } - ] - } - })) - .expect("standard recreate view request should parse"); - let recreated = create_view_response(&store, &metadata_backend, "warehouse", &namespace, recreate_request, true) - .await - .expect("dropped view name should be reusable"); - assert_ne!(recreated.metadata_location, created.metadata_location); - } - - #[tokio::test] - async fn table_ref_write_responses_commit_retention_refs_and_protect_deletes() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let manifest_list = format!("{table_location}/metadata/snap-10.avro"); - let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; - - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 10, - "sequence-number": 1, - "timestamp-ms": 1234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 10, - "type": "branch" - } - ] - })) - .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append should commit"); - - let ref_request: PutTableRefRequest = serde_json::from_value(serde_json::json!({ - "snapshot-id": 10, - "type": "tag", - "max-ref-age-ms": 86400000, - "expected-snapshot-id": null - })) - .expect("ref put request should parse"); - put_table_ref_response(&store, &metadata_backend, "warehouse", &namespace, "events", "audit", ref_request) - .await - .expect("ref put should commit"); - - let refs = table_refs_response(&store, &metadata_backend, "warehouse", &namespace, "events") - .await - .expect("refs should load"); - assert_eq!(refs.refs["audit"]["type"], "tag"); - assert_eq!(refs.refs["audit"]["max-ref-age-ms"], 86400000); - - let delete_without_force: DeleteTableRefRequest = - serde_json::from_value(serde_json::json!({})).expect("ref delete request should parse"); - let error = delete_table_ref_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - "events", - "audit", - delete_without_force, - ) - .await - .expect_err("retention refs should require force delete"); - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - - let force_delete: DeleteTableRefRequest = - serde_json::from_value(serde_json::json!({ "force": true })).expect("ref force delete should parse"); - delete_table_ref_response(&store, &metadata_backend, "warehouse", &namespace, "events", "audit", force_delete) - .await - .expect("force delete should commit"); - let refs = table_refs_response(&store, &metadata_backend, "warehouse", &namespace, "events") - .await - .expect("refs should load after delete"); - assert!(!refs.refs.contains_key("audit")); - - let main_delete: DeleteTableRefRequest = - serde_json::from_value(serde_json::json!({ "force": true })).expect("main delete request should parse"); - let error = delete_table_ref_response(&store, &metadata_backend, "warehouse", &namespace, "events", "main", main_delete) - .await - .expect_err("main ref should remain protected"); - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - } - - #[test] - fn load_table_response_includes_rest_metadata_payload() { - let metadata = serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id", - "metadata-log": [ - { - "timestamp-ms": 1, - "metadata-file": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json" - }, - { - "timestamp-ms": 2, - "metadata-file": "s3://warehouse/external/metadata/00000.metadata.json" - } - ] - }); - let response = load_table_response_from_entry(table_entry_for_credentials(), metadata); - - assert_eq!( - response.metadata_location, - "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json" - ); - assert_eq!( - response.metadata["metadata-log"][0]["metadata-file"], - "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json" - ); - assert_eq!( - response.metadata["metadata-log"][1]["metadata-file"], - "s3://warehouse/external/metadata/00000.metadata.json" - ); - assert!(response.storage_credentials.is_empty()); - assert_eq!(response.config.get("rustfs.credential-vending"), Some(&"unsupported".to_string())); - assert_eq!( - response.config.get("rustfs.credential-vending-reason"), - Some(&"temporary-credentials-not-implemented".to_string()) - ); - assert_eq!(response.config.get("rustfs.credential-scope"), Some(&"table-prefix".to_string())); - assert_eq!( - response.config.get("rustfs.credential-scope-prefix"), - Some(&"s3://warehouse/tables/table-id".to_string()) - ); - assert_eq!( - response.config.get("rustfs.credential-mode"), - Some(&"client-provided-s3-credentials-required".to_string()) - ); - assert!(!response.config.contains_key("s3.access-key-id")); - assert!(!response.config.contains_key("s3.secret-access-key")); - assert!(!response.config.contains_key("s3.session-token")); - } - - #[test] - fn load_table_response_preserves_format_v4_relative_metadata_log() { - let metadata = serde_json::json!({ - "format-version": 4, - "table-uuid": "table-uuid", - "metadata-log": [ - { - "timestamp-ms": 1, - "metadata-file": "metadata/00000.metadata.json" - }, - { - "timestamp-ms": 2, - "metadata-file": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json" - } - ] - }); - - let response = load_table_response_from_entry(table_entry_for_credentials(), metadata); - - assert_eq!(response.metadata["metadata-log"][0]["metadata-file"], "metadata/00000.metadata.json"); - assert_eq!( - response.metadata["metadata-log"][1]["metadata-file"], - "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json" - ); - } - - #[test] - fn table_metadata_location_for_catalog_accepts_only_the_table_bucket() { - let object_key = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - - assert_eq!( - table_metadata_location_for_catalog("warehouse", &format!("s3://warehouse/{object_key}")) - .expect("same-bucket metadata location should normalize"), - object_key - ); - assert!(table_metadata_location_for_catalog("warehouse", &format!("s3://other/{object_key}")).is_err()); - } - - fn table_entry_for_credentials() -> crate::table_catalog::TableEntry { - crate::table_catalog::TableEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "warehouse".to_string(), - namespace: "analytics".to_string(), - table: "events".to_string(), - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: "s3://warehouse/tables/table-id".to_string(), - metadata_location: ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json" - .to_string(), - version_token: "token-v1".to_string(), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - } - } - - #[tokio::test] - async fn disabled_table_credential_issuer_keeps_credentials_empty() { - let issuer = DisabledTableCredentialIssuer; - let response = load_credentials_response_from_entry(&table_entry_for_credentials(), &issuer, None) - .await - .expect("disabled issuer should build an empty response"); - - assert!(response.storage_credentials.is_empty()); - assert_eq!( - response.config.get(CREDENTIAL_VENDING_CONFIG_KEY), - Some(&CREDENTIAL_VENDING_UNSUPPORTED.to_string()) - ); - assert_eq!( - response.config.get(CREDENTIAL_MODE_CONFIG_KEY), - Some(&CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()) - ); - assert_eq!( - response.config.get(CREDENTIAL_VENDING_REASON_CONFIG_KEY), - Some(&"credential-vending-disabled".to_string()) - ); - } - - #[tokio::test] - async fn disabled_table_credential_issuer_skips_scope_validation() { - let issuer = DisabledTableCredentialIssuer; - let mut entry = table_entry_for_credentials(); - entry.warehouse_location = "s3://warehouse/".to_string(); - - let response = load_credentials_response_from_entry(&entry, &issuer, None) - .await - .expect("disabled issuer should not validate credential scopes"); - - assert!(response.storage_credentials.is_empty()); - assert_eq!( - response.config.get(CREDENTIAL_VENDING_CONFIG_KEY), - Some(&CREDENTIAL_VENDING_UNSUPPORTED.to_string()) - ); - assert!(!response.config.contains_key(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY)); - } - - struct UnavailableTableCredentialIssuer; - - #[async_trait::async_trait] - impl TableCredentialIssuer for UnavailableTableCredentialIssuer { - async fn issue_table_credentials( - &self, - request: TableCredentialIssueRequest<'_>, - ) -> S3Result> { - assert_eq!(request.scope_prefix, "s3://warehouse/tables/table-id/"); - Ok(None) - } - } - - #[tokio::test] - async fn unavailable_table_credential_issuer_reports_fallback_scope() { - let issuer = UnavailableTableCredentialIssuer; - let response = load_credentials_response_from_entry(&table_entry_for_credentials(), &issuer, None) - .await - .expect("unavailable issuer should build a fallback response"); - - assert!(response.storage_credentials.is_empty()); - assert_eq!( - response.config.get(CREDENTIAL_VENDING_CONFIG_KEY), - Some(&CREDENTIAL_VENDING_UNSUPPORTED.to_string()) - ); - assert_eq!( - response.config.get(CREDENTIAL_MODE_CONFIG_KEY), - Some(&CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()) - ); - assert_eq!( - response.config.get(CREDENTIAL_VENDING_REASON_CONFIG_KEY), - Some(&CREDENTIAL_VENDING_UNSUPPORTED_REASON.to_string()) - ); - assert_eq!( - response.config.get(CREDENTIAL_SCOPE_CONFIG_KEY), - Some(&CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()) - ); - assert_eq!( - response.config.get(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY), - Some(&"s3://warehouse/tables/table-id/".to_string()) - ); - } - - struct TestTableCredentialIssuer; - - #[async_trait::async_trait] - impl TableCredentialIssuer for TestTableCredentialIssuer { - async fn issue_table_credentials( - &self, - request: TableCredentialIssueRequest<'_>, - ) -> S3Result> { - assert_eq!(request.entry.table_bucket, "warehouse"); - assert_eq!(request.scope_prefix, "s3://warehouse/tables/table-id/"); - assert_eq!(request.object_prefix, "tables/table-id/"); - Ok(Some(IssuedTableCredentials { - access_key_id: "temporary-access-key".to_string(), - secret_access_key: "temporary-secret-key".to_string(), - session_token: "temporary-session-token".to_string(), - expiration: OffsetDateTime::from_unix_timestamp(1_800_000_000).expect("test timestamp should be valid"), - })) - } - } - - #[tokio::test] - async fn credential_issuer_returns_temporary_scoped_storage_credentials() { - let issuer = TestTableCredentialIssuer; - let principal = rustfs_credentials::Credentials { - access_key: "parent-access-key".to_string(), - secret_key: "parent-secret-key".to_string(), - ..Default::default() - }; - - let response = load_credentials_response_from_entry(&table_entry_for_credentials(), &issuer, Some(&principal)) - .await - .expect("issuer should build a scoped credential response"); - - assert_eq!( - response.config.get(CREDENTIAL_VENDING_CONFIG_KEY), - Some(&CREDENTIAL_VENDING_SUPPORTED.to_string()) - ); - assert_eq!( - response.config.get(CREDENTIAL_MODE_CONFIG_KEY), - Some(&CREDENTIAL_MODE_CATALOG_VENDED.to_string()) - ); - assert!(!response.config.contains_key(S3_ACCESS_KEY_ID_CONFIG_KEY)); - assert!(!response.config.contains_key(S3_SECRET_ACCESS_KEY_CONFIG_KEY)); - assert!(!response.config.contains_key(S3_SESSION_TOKEN_CONFIG_KEY)); - assert_eq!(response.storage_credentials.len(), 1); - let credential = &response.storage_credentials[0]; - assert_eq!(credential.prefix, "s3://warehouse/tables/table-id/"); - assert_eq!(credential.config.get("s3.access-key-id"), Some(&"temporary-access-key".to_string())); - assert_eq!(credential.config.get("s3.secret-access-key"), Some(&"temporary-secret-key".to_string())); - assert_eq!(credential.config.get("s3.session-token"), Some(&"temporary-session-token".to_string())); - assert_eq!( - credential.config.get("rustfs.credential-mode"), - Some(&"catalog-vended-temporary-credentials".to_string()) - ); - assert_eq!( - credential.config.get("rustfs.credential-scope-prefix"), - Some(&"s3://warehouse/tables/table-id/".to_string()) - ); - assert_eq!( - credential.config.get("rustfs.credential-expiration-unix-seconds"), - Some(&"1800000000".to_string()) - ); - assert!(!credential.config.contains_key("rustfs.credential-vending-reason")); - } - - #[tokio::test] - async fn credential_response_serializes_sensitive_config_only_inside_storage_credentials() { - let issuer = TestTableCredentialIssuer; - let response = load_credentials_response_from_entry(&table_entry_for_credentials(), &issuer, None) - .await - .expect("issuer should build a scoped credential response"); - - let value = serde_json::to_value(&response).expect("credential response should serialize"); - - assert_eq!( - value["config"][CREDENTIAL_VENDING_CONFIG_KEY], - serde_json::Value::String(CREDENTIAL_VENDING_SUPPORTED.to_string()) - ); - assert!(value["config"].get(S3_ACCESS_KEY_ID_CONFIG_KEY).is_none()); - assert!(value["config"].get(S3_SECRET_ACCESS_KEY_CONFIG_KEY).is_none()); - assert!(value["config"].get(S3_SESSION_TOKEN_CONFIG_KEY).is_none()); - assert_eq!( - value["storage-credentials"][0]["config"][S3_ACCESS_KEY_ID_CONFIG_KEY], - serde_json::Value::String("temporary-access-key".to_string()) - ); - } - - #[test] - fn table_credentials_do_not_snapshot_parent_groups() { - let principal = rustfs_credentials::Credentials { - access_key: "parent-access-key".to_string(), - groups: Some(vec!["analytics-writers".to_string()]), - ..Default::default() - }; - let mut credential = rustfs_credentials::Credentials::default(); - - bind_table_credential_parent(&mut credential, &principal); - - assert_eq!(credential.parent_user, "parent-access-key"); - assert!(credential.groups.is_none()); - } - - #[tokio::test] - async fn table_credential_session_policy_is_limited_to_table_prefix() { - let policy = table_credential_session_policy(&table_entry_for_credentials(), "tables/table-id/") - .expect("table credential policy should build"); - let groups = None; - let conditions = std::collections::HashMap::new(); - let claims = std::collections::HashMap::new(); - - assert!( - policy - .is_allowed(&rustfs_policy::policy::Args { - account: "temporary-access-key", - groups: &groups, - action: Action::S3Action(rustfs_policy::policy::action::S3Action::GetObjectAction), - bucket: "warehouse", - conditions: &conditions, - is_owner: false, - object: "tables/table-id/data/file.parquet", - claims: &claims, - deny_only: false, - }) - .await - ); - assert!( - policy - .is_allowed(&rustfs_policy::policy::Args { - account: "temporary-access-key", - groups: &groups, - action: Action::S3Action(rustfs_policy::policy::action::S3Action::GetBucketLocationAction), - bucket: "warehouse", - conditions: &conditions, - is_owner: false, - object: "", - claims: &claims, - deny_only: false, - }) - .await - ); - assert!( - policy - .is_allowed(&rustfs_policy::policy::Args { - account: "temporary-access-key", - groups: &groups, - action: Action::AdminAction(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction), - bucket: "warehouse", - conditions: &conditions, - is_owner: false, - object: "namespaces/analytics/tables/events", - claims: &claims, - deny_only: false, - }) - .await - ); - assert!( - !policy - .is_allowed(&rustfs_policy::policy::Args { - account: "temporary-access-key", - groups: &groups, - action: Action::S3Action(rustfs_policy::policy::action::S3Action::GetObjectAction), - bucket: "warehouse", - conditions: &conditions, - is_owner: false, - object: "tables/other/data/file.parquet", - claims: &claims, - deny_only: false, - }) - .await - ); - assert!( - !policy - .is_allowed(&rustfs_policy::policy::Args { - account: "temporary-access-key", - groups: &groups, - action: Action::S3Action(rustfs_policy::policy::action::S3Action::PutObjectAction), - bucket: "other-warehouse", - conditions: &conditions, - is_owner: false, - object: "tables/table-id/data/file.parquet", - claims: &claims, - deny_only: false, - }) - .await - ); - } - - #[tokio::test] - async fn table_credential_session_policy_includes_table_resource_actions() { - let policy = table_credential_session_policy(&table_entry_for_credentials(), "tables/table-id/") - .expect("table credential policy should build"); - let groups = None; - let conditions = std::collections::HashMap::new(); - let claims = std::collections::HashMap::new(); - - assert!( - policy - .is_allowed(&rustfs_policy::policy::Args { - account: "temporary-access-key", - groups: &groups, - action: Action::AdminAction(rustfs_policy::policy::action::AdminAction::GetTableMetadataAction), - bucket: "warehouse", - conditions: &conditions, - is_owner: false, - object: "namespaces/analytics/tables/events", - claims: &claims, - deny_only: false, - }) - .await - ); - assert!( - !policy - .is_allowed(&rustfs_policy::policy::Args { - account: "temporary-access-key", - groups: &groups, - action: Action::AdminAction(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction), - bucket: "warehouse", - conditions: &conditions, - is_owner: false, - object: "namespaces/analytics/tables/orders", - claims: &claims, - deny_only: false, - }) - .await - ); - assert!( - !policy - .is_allowed(&rustfs_policy::policy::Args { - account: "temporary-access-key", - groups: &groups, - action: Action::AdminAction(rustfs_policy::policy::action::AdminAction::CreateTableAction), - bucket: "warehouse", - conditions: &conditions, - is_owner: false, - object: "namespaces/analytics/tables/events", - claims: &claims, - deny_only: false, - }) - .await - ); - } - - #[test] - fn table_credential_scope_rejects_cross_bucket_or_unsafe_prefix() { - let mut entry = table_entry_for_credentials(); - entry.warehouse_location = "s3://other-warehouse/tables/table-id".to_string(); - assert!(table_credential_scope(&entry).is_err()); - - let mut entry = table_entry_for_credentials(); - entry.warehouse_location = "s3://warehouse/tables/../table-id".to_string(); - assert!(table_credential_scope(&entry).is_err()); - } - - #[test] - fn commit_table_request_uses_rest_commit_fields() { - let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "commit-id": "commit-1", - "idempotency-key": "retry-1", - "operation": "append", - "expected-version-token": "token-v1", - "expected-metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", - "new-metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json", - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "writer": "pyiceberg" - })) - .expect("commit request should parse"); - - assert_eq!(request.commit_id.as_deref(), Some("commit-1")); - assert_eq!(request.idempotency_key.as_deref(), Some("retry-1")); - assert_eq!(request.operation.as_deref(), Some("append")); - assert_eq!(request.expected_version_token.as_deref(), Some("token-v1")); - assert_eq!( - request.new_metadata_location.as_deref(), - Some(".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json") - ); - assert_eq!(request.requirements.len(), 1); - assert_eq!(request.writer.as_deref(), Some("pyiceberg")); - } - - #[derive(Default)] - struct TestTableCatalogStore { - table_buckets: tokio::sync::Mutex>, - namespaces: tokio::sync::Mutex>, - tables: tokio::sync::Mutex>, - views: tokio::sync::Mutex>, - commits: tokio::sync::Mutex>, - fail_put_table_bucket: tokio::sync::Mutex, - } - - #[derive(Clone, Default)] - struct TestTableCatalogObjectBackend { - objects: Arc>>, - put_object_barrier: Option>, - } - - impl TestTableCatalogObjectBackend { - async fn put_bytes(&self, bucket: &str, object: &str, data: Vec) { - self.objects.lock().await.insert( - (bucket.to_string(), object.to_string()), - crate::table_catalog::TableCatalogObject { - data, - etag: Some("etag".to_string()), - mod_time: None, - }, - ); - } - - async fn put_json(&self, bucket: &str, object: &str, value: serde_json::Value) { - self.put_json_with_mod_time(bucket, object, value, None).await; - } - - async fn put_json_with_mod_time( - &self, - bucket: &str, - object: &str, - value: serde_json::Value, - mod_time: Option, - ) { - let data = serde_json::to_vec(&value).expect("metadata JSON should serialize"); - self.objects.lock().await.insert( - (bucket.to_string(), object.to_string()), - crate::table_catalog::TableCatalogObject { - data, - etag: Some("etag".to_string()), - mod_time, - }, - ); - } - } - - fn test_snapshot_object_key(bucket: &str, location: &str) -> String { - crate::table_catalog::table_catalog_object_key_from_location(bucket, location) - .expect("test snapshot object location should be valid") - } - - fn test_manifest_list_avro_bytes(manifest_paths: &[&str], sequence_number: i64, snapshot_id: i64) -> Vec { - let manifests = manifest_paths - .iter() - .map(|manifest_path| (*manifest_path, sequence_number, snapshot_id)) - .collect::>(); - test_manifest_list_avro_entries(&manifests) - } - - fn test_manifest_list_avro_entries(manifests: &[(&str, i64, i64)]) -> Vec { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "manifest_file", - "fields": [ - {"name": "manifest_path", "type": "string"}, - {"name": "sequence_number", "type": "long"}, - {"name": "added_snapshot_id", "type": "long"} - ] - } - "#, - ) - .expect("manifest list avro schema should parse"); - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - for (manifest_path, sequence_number, snapshot_id) in manifests { - writer - .append(apache_avro::types::Value::Record(vec![ - ( - "manifest_path".to_string(), - apache_avro::types::Value::String((*manifest_path).to_string()), - ), - ("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), - ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), - ])) - .expect("manifest list record should append"); - } - writer.into_inner().expect("manifest list avro bytes should flush") - } - - fn test_manifest_avro_bytes(files: &[(&str, i32, i32, i64, i64)]) -> Vec { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "manifest_entry", - "fields": [ - {"name": "status", "type": "int"}, - {"name": "snapshot_id", "type": "long"}, - {"name": "sequence_number", "type": "long"}, - {"name": "file_sequence_number", "type": "long"}, - { - "name": "data_file", - "type": { - "type": "record", - "name": "data_file", - "fields": [ - {"name": "content", "type": "int"}, - {"name": "file_path", "type": "string"}, - {"name": "record_count", "type": "long"}, - {"name": "file_size_in_bytes", "type": "long"} - ] - } - } - ] - } - "#, - ) - .expect("manifest avro schema should parse"); - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - for (file_path, content, status, snapshot_id, sequence_number) in files { - writer - .append(apache_avro::types::Value::Record(vec![ - ("status".to_string(), apache_avro::types::Value::Int(*status)), - ("snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), - ("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), - ("file_sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), - ( - "data_file".to_string(), - apache_avro::types::Value::Record(vec![ - ("content".to_string(), apache_avro::types::Value::Int(*content)), - ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), - ("record_count".to_string(), apache_avro::types::Value::Long(1)), - ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), - ]), - ), - ])) - .expect("manifest record should append"); - } - writer.into_inner().expect("manifest avro bytes should flush") - } - - fn test_nullable_long(value: Option) -> apache_avro::types::Value { - match value { - Some(value) => apache_avro::types::Value::Union(1, Box::new(apache_avro::types::Value::Long(value))), - None => apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - } - } - - fn test_manifest_avro_bytes_with_nullable_sequences(files: &[(&str, i32, i32, i64, Option)]) -> Vec { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "manifest_entry", - "fields": [ - {"name": "status", "type": "int"}, - {"name": "snapshot_id", "type": "long"}, - {"name": "sequence_number", "type": ["null", "long"], "default": null}, - {"name": "file_sequence_number", "type": ["null", "long"], "default": null}, - { - "name": "data_file", - "type": { - "type": "record", - "name": "data_file", - "fields": [ - {"name": "content", "type": "int"}, - {"name": "file_path", "type": "string"}, - {"name": "record_count", "type": "long"}, - {"name": "file_size_in_bytes", "type": "long"} - ] - } - } - ] - } - "#, - ) - .expect("manifest avro schema should parse"); - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - for (file_path, content, status, snapshot_id, sequence_number) in files { - writer - .append(apache_avro::types::Value::Record(vec![ - ("status".to_string(), apache_avro::types::Value::Int(*status)), - ("snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), - ("sequence_number".to_string(), test_nullable_long(*sequence_number)), - ("file_sequence_number".to_string(), test_nullable_long(*sequence_number)), - ( - "data_file".to_string(), - apache_avro::types::Value::Record(vec![ - ("content".to_string(), apache_avro::types::Value::Int(*content)), - ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), - ("record_count".to_string(), apache_avro::types::Value::Long(1)), - ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), - ]), - ), - ])) - .expect("manifest record should append"); - } - writer.into_inner().expect("manifest avro bytes should flush") - } - - async fn seed_test_manifest_list( - backend: &TestTableCatalogObjectBackend, - bucket: &str, - manifest_list_location: &str, - manifest_locations: &[&str], - sequence_number: i64, - snapshot_id: i64, - ) { - let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); - backend - .put_bytes( - bucket, - &manifest_list_key, - test_manifest_list_avro_bytes(manifest_locations, sequence_number, snapshot_id), - ) - .await; - } - - async fn seed_test_snapshot_manifest( - backend: &TestTableCatalogObjectBackend, - bucket: &str, - manifest_list_location: &str, - snapshot_id: i64, - sequence_number: i64, - files: &[(&str, i32, i32, i64, i64)], - ) { - let manifest_location = manifest_list_location - .rsplit_once('/') - .map(|(prefix, name)| format!("{prefix}/manifest-{name}")) - .expect("manifest list location should include a file name"); - let manifest_key = test_snapshot_object_key(bucket, &manifest_location); - let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); - backend - .put_bytes( - bucket, - &manifest_list_key, - test_manifest_list_avro_bytes(&[&manifest_location], sequence_number, snapshot_id), - ) - .await; - backend - .put_bytes(bucket, &manifest_key, test_manifest_avro_bytes(files)) - .await; - seed_test_manifest_data_files(backend, bucket, files).await; - } - - async fn seed_test_manifest_with_nullable_sequences( - backend: &TestTableCatalogObjectBackend, - bucket: &str, - manifest_location: &str, - files: &[(&str, i32, i32, i64, Option)], - ) { - let manifest_key = test_snapshot_object_key(bucket, manifest_location); - backend - .put_bytes(bucket, &manifest_key, test_manifest_avro_bytes_with_nullable_sequences(files)) - .await; - let data_files = files - .iter() - .map(|(file_path, content, status, snapshot_id, sequence_number)| { - (*file_path, *content, *status, *snapshot_id, sequence_number.unwrap_or_default()) - }) - .collect::>(); - seed_test_manifest_data_files(backend, bucket, &data_files).await; - } - - async fn seed_test_manifest( - backend: &TestTableCatalogObjectBackend, - bucket: &str, - manifest_location: &str, - files: &[(&str, i32, i32, i64, i64)], - ) { - let manifest_key = test_snapshot_object_key(bucket, manifest_location); - backend - .put_bytes(bucket, &manifest_key, test_manifest_avro_bytes(files)) - .await; - seed_test_manifest_data_files(backend, bucket, files).await; - } - - async fn seed_test_manifest_data_files( - backend: &TestTableCatalogObjectBackend, - bucket: &str, - files: &[(&str, i32, i32, i64, i64)], - ) { - for (file_path, _, status, _, _) in files { - if *status != 2 { - let object_key = test_snapshot_object_key(bucket, file_path); - backend.put_bytes(bucket, &object_key, b"data".to_vec()).await; - } - } - } - - async fn create_standard_events_table( - store: &TestTableCatalogStore, - metadata_backend: &TestTableCatalogObjectBackend, - namespace: &crate::table_catalog::Namespace, - ) -> RestLoadTableResponse { - ensure_table_bucket_entry(store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - store, - "warehouse", - CreateNamespaceRequest { - namespace: namespace.public_name().split('.').map(str::to_string).collect(), - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let create_request: CreateTableRequest = serde_json::from_value(serde_json::json!({ - "name": "events", - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "id", - "required": true, - "type": "long" - } - ] - } - })) - .expect("standard create table request should parse"); - create_table_response(store, metadata_backend, "warehouse", namespace, create_request, true) - .await - .expect("table should be created") - } - - async fn seed_object_table_for_metadata_maintenance( - store: &crate::table_catalog::ObjectTableCatalogStore, - backend: &TestTableCatalogObjectBackend, - bucket: &str, - namespace: &crate::table_catalog::Namespace, - table: &crate::table_catalog::IdentifierSegment, - current_metadata_location: String, - ) { - store - .put_table_bucket(crate::table_catalog::TableBucketEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - catalog_type: crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(), - warehouse_root: format!("s3://{bucket}/"), - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }) - .await - .expect("table bucket entry should seed"); - store - .create_namespace(crate::table_catalog::NamespaceEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - namespace_id: namespace.storage_id(), - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }) - .await - .expect("namespace entry should seed"); - store - .create_table(crate::table_catalog::TableEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: format!("s3://{bucket}/tables/table-id"), - metadata_location: current_metadata_location, - version_token: "token-v1".to_string(), - generation: 1, - state: crate::table_catalog::TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }) - .await - .expect("table entry should seed"); - backend - .put_json(bucket, "unrelated/ignored.json", serde_json::json!({})) - .await; - } - - #[async_trait::async_trait] - impl crate::table_catalog::TableCatalogObjectBackend for TestTableCatalogObjectBackend { - async fn read_object( - &self, - bucket: &str, - object: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .objects - .lock() - .await - .get(&(bucket.to_string(), object.to_string())) - .cloned()) - } - - async fn object_exists(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult { - Ok(self - .objects - .lock() - .await - .contains_key(&(bucket.to_string(), object.to_string()))) - } - - async fn put_object( - &self, - bucket: &str, - object: &str, - data: Vec, - precondition: crate::table_catalog::TableCatalogPutPrecondition, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - let key = (bucket.to_string(), object.to_string()); - let mut objects = self.objects.lock().await; - let result = if matches!(precondition, crate::table_catalog::TableCatalogPutPrecondition::IfAbsent) - && objects.contains_key(&key) - { - Err(crate::table_catalog::TableCatalogStoreError::Conflict(format!( - "object already exists: {object}" - ))) - } else { - objects.insert( - key, - crate::table_catalog::TableCatalogObject { - data, - etag: Some("etag".to_string()), - mod_time: None, - }, - ); - Ok(()) - }; - drop(objects); - if let Some(barrier) = &self.put_object_barrier { - barrier.wait().await; - } - result - } - - async fn delete_object(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { - self.objects.lock().await.remove(&(bucket.to_string(), object.to_string())); - Ok(()) - } - - async fn list_objects(&self, bucket: &str, prefix: &str) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .objects - .lock() - .await - .keys() - .filter(|(object_bucket, object)| object_bucket == bucket && object.starts_with(prefix)) - .map(|(_, object)| object.clone()) - .collect()) - } - - async fn acquire_write_lock( - &self, - _bucket: &str, - _object: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(Box::new(())) - } - } - - #[async_trait::async_trait] - impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { - async fn get_table_bucket( - &self, - table_bucket: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .table_buckets - .lock() - .await - .iter() - .find(|entry| entry.table_bucket == table_bucket) - .cloned()) - } - - async fn put_table_bucket( - &self, - entry: crate::table_catalog::TableBucketEntry, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - let mut fail_put_table_bucket = self.fail_put_table_bucket.lock().await; - if *fail_put_table_bucket { - *fail_put_table_bucket = false; - return Err(crate::table_catalog::TableCatalogStoreError::Internal( - "injected table bucket write failure".to_string(), - )); - } - drop(fail_put_table_bucket); - - let mut table_buckets = self.table_buckets.lock().await; - table_buckets.retain(|existing| existing.table_bucket != entry.table_bucket); - table_buckets.push(entry); - Ok(()) - } - - async fn create_namespace( - &self, - entry: crate::table_catalog::NamespaceEntry, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - if self.get_table_bucket(&entry.table_bucket).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table bucket {}", - entry.table_bucket - ))); - } - self.namespaces.lock().await.push(entry); - Ok(()) - } - - async fn list_namespaces( - &self, - table_bucket: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .namespaces - .lock() - .await - .iter() - .filter(|entry| entry.table_bucket == table_bucket) - .cloned() - .collect()) - } - - async fn get_namespace( - &self, - table_bucket: &str, - namespace: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .namespaces - .lock() - .await - .iter() - .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) - .cloned()) - } - - async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { - self.namespaces - .lock() - .await - .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace)); - Ok(()) - } - - async fn create_table( - &self, - entry: crate::table_catalog::TableEntry, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - if self.get_table_bucket(&entry.table_bucket).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table bucket {}", - entry.table_bucket - ))); - } - if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - self.tables.lock().await.push(entry); - Ok(()) - } - - async fn register_table( - &self, - entry: crate::table_catalog::TableEntry, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - if self.get_table_bucket(&entry.table_bucket).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table bucket {}", - entry.table_bucket - ))); - } - if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - self.tables.lock().await.push(entry); - Ok(()) - } - - async fn list_tables( - &self, - table_bucket: &str, - namespace: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .tables - .lock() - .await - .iter() - .filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) - .cloned() - .collect()) - } - - async fn load_table( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .tables - .lock() - .await - .iter() - .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table) - .cloned()) - } - - async fn commit_table( - &self, - request: crate::table_catalog::TableCommitRequest, - ) -> crate::table_catalog::TableCatalogStoreResult { - let mut tables = self.tables.lock().await; - let Some(index) = tables.iter().position(|entry| { - entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.table == request.table - }) else { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - request.table_bucket, request.namespace, request.table - ))); - }; - - let current = tables[index].clone(); - if current.version_token != request.expected_version_token { - return Err(crate::table_catalog::TableCatalogStoreError::Conflict( - "current table version token does not match expected token".to_string(), - )); - } - if current.metadata_location != request.expected_metadata_location { - return Err(crate::table_catalog::TableCatalogStoreError::Conflict( - "current table metadata location does not match expected location".to_string(), - )); - } - - let mut next = current.clone(); - next.metadata_location = request.new_metadata_location.clone(); - next.version_token = "token-committed".to_string(); - next.generation = next.generation.saturating_add(1); - tables[index] = next.clone(); - drop(tables); - - let commit_log = crate::table_catalog::CommitLogEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - commit_id: request.commit_id, - idempotency_key: request.idempotency_key, - table_id: current.table_id, - operation: request.operation, - expected_version_token: request.expected_version_token, - new_version_token: next.version_token.clone(), - previous_metadata_location: request.expected_metadata_location, - new_metadata_location: request.new_metadata_location, - requirements: request.requirements, - status: crate::table_catalog::CommitLogStatus::Committed, - writer: request.writer, - created_at: None, - updated_at: None, - }; - self.commits.lock().await.push(commit_log.clone()); - - Ok(crate::table_catalog::TableCommitResult { table: next, commit_log }) - } - - async fn drop_table( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - self.tables - .lock() - .await - .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table)); - Ok(()) - } - - async fn create_view(&self, entry: crate::table_catalog::ViewEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { - if self.get_table_bucket(&entry.table_bucket).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table bucket {}", - entry.table_bucket - ))); - } - if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - self.views.lock().await.push(entry); - Ok(()) - } - - async fn list_views( - &self, - table_bucket: &str, - namespace: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .views - .lock() - .await - .iter() - .filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) - .cloned() - .collect()) - } - - async fn load_view( - &self, - table_bucket: &str, - namespace: &str, - view: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .views - .lock() - .await - .iter() - .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view) - .cloned()) - } - - async fn replace_view( - &self, - request: crate::table_catalog::ViewCommitRequest, - ) -> crate::table_catalog::TableCatalogStoreResult { - let mut views = self.views.lock().await; - let Some(index) = views.iter().position(|entry| { - entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.view == request.view - }) else { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "view {}/{}/{}", - request.table_bucket, request.namespace, request.view - ))); - }; - let current = views[index].clone(); - if current.version_token != request.expected_version_token { - return Err(crate::table_catalog::TableCatalogStoreError::Conflict( - "current view version token does not match expected token".to_string(), - )); - } - if current.metadata_location != request.expected_metadata_location { - return Err(crate::table_catalog::TableCatalogStoreError::Conflict( - "current view metadata location does not match expected location".to_string(), - )); - } - let mut next = current; - next.metadata_location = request.new_metadata_location; - next.version_token = "token-view-committed".to_string(); - next.generation = next.generation.saturating_add(1); - views[index] = next.clone(); - Ok(crate::table_catalog::ViewCommitResult { view: next }) - } - - async fn drop_view( - &self, - table_bucket: &str, - namespace: &str, - view: &str, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - self.views - .lock() - .await - .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view)); - Ok(()) - } - - async fn get_commit_by_id( - &self, - _table_bucket: &str, - _table_id: &str, - _commit_id: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(None) - } - - async fn get_commit_by_idempotency_key( - &self, - _table_bucket: &str, - _table_id: &str, - _idempotency_key: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(None) - } - } - - #[tokio::test] - async fn ensure_table_bucket_entry_seeds_enabled_bucket_before_namespace_create() { - let store = TestTableCatalogStore::default(); - - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - let table_bucket = store - .get_table_bucket("warehouse") - .await - .expect("table bucket lookup should succeed") - .expect("table bucket entry should exist"); - - assert_eq!(table_bucket.table_bucket, "warehouse"); - assert_eq!(table_bucket.catalog_type, crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE); - assert_eq!(table_bucket.warehouse_root, "s3://warehouse/"); - } - - #[tokio::test] - async fn ensure_table_bucket_entry_rejects_bucket_without_table_marker() { - let store = TestTableCatalogStore::default(); - - assert!(ensure_table_bucket_entry(&store, "warehouse", false).await.is_err()); - assert!( - store - .get_table_bucket("warehouse") - .await - .expect("table bucket lookup should succeed") - .is_none() - ); - } - - #[tokio::test] - async fn ensure_table_bucket_entry_propagates_catalog_entry_failure() { - let store = TestTableCatalogStore::default(); - *store.fail_put_table_bucket.lock().await = true; - - assert!(ensure_table_bucket_entry(&store, "warehouse", true).await.is_err()); - assert!( - store - .get_table_bucket("warehouse") - .await - .expect("table bucket lookup should succeed") - .is_none() - ); - assert!(!*store.fail_put_table_bucket.lock().await); - } - - #[tokio::test] - async fn namespace_helpers_call_catalog_store() { - let store = TestTableCatalogStore::default(); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - let create = create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), - }, - true, - ) - .await - .expect("namespace should be created"); - - assert_eq!(create.namespace, vec!["analytics".to_string()]); - assert_eq!(create.properties.get("owner").map(String::as_str), Some("lakehouse")); - - let unpaginated_uri = "/".parse::().expect("list URI should parse"); - let list = list_namespaces_response(&store, "warehouse", &unpaginated_uri) - .await - .expect("namespace list should load"); - assert_eq!(list.namespaces, vec![vec!["analytics".to_string()]]); - - drop_namespace_in_store(&store, "warehouse", "analytics") - .await - .expect("namespace should drop"); - let list = list_namespaces_response(&store, "warehouse", &unpaginated_uri) - .await - .expect("namespace list should load after drop"); - assert!(list.namespaces.is_empty()); - } - - #[tokio::test] - async fn table_helpers_call_catalog_store() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - - let metadata_location = - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - metadata_backend - .put_json( - "warehouse", - metadata_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - let register = register_table_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: metadata_location.to_string(), - overwrite: false, - }, - true, - ) - .await - .expect("table should register"); - - let client_metadata_location = table_metadata_location_for_client("warehouse", metadata_location); - assert_eq!(register.metadata_location, client_metadata_location); - assert_eq!(register.metadata["format-version"], 2); - - let unpaginated_uri = "/".parse::().expect("list URI should parse"); - let list = list_tables_response(&store, "warehouse", &namespace, &unpaginated_uri) - .await - .expect("table list should load"); - assert_eq!(list.identifiers[0].name, "events"); - - let load = load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events") - .await - .expect("table should load"); - assert_eq!(load.metadata_location, client_metadata_location); - assert_eq!(load.metadata["table-uuid"], "table-uuid"); - - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let next_metadata_location = - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; - metadata_backend - .put_json( - "warehouse", - next_metadata_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id", - "last-sequence-number": 2 - }), - ) - .await; - let commit = commit_table_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - "events", - RestCommitTableRequest { - commit_id: Some("commit-1".to_string()), - idempotency_key: Some("retry-1".to_string()), - operation: Some("append".to_string()), - expected_version_token: Some(current.version_token.clone()), - expected_metadata_location: Some(table_metadata_location_for_client("warehouse", ¤t.metadata_location)), - new_metadata_location: Some(table_metadata_location_for_client("warehouse", next_metadata_location)), - requirements: Vec::new(), - updates: Vec::new(), - _identifier: None, - writer: Some("pyiceberg".to_string()), - }, - ) - .await - .expect("table commit should succeed"); - assert_eq!( - commit.metadata_location, - table_metadata_location_for_client("warehouse", next_metadata_location) - ); - assert_eq!(commit.version_token, "token-committed"); - assert_eq!(commit.generation, current.generation + 1); - assert_eq!(commit.commit_id, "commit-1"); - - let committed = store - .load_table("warehouse", "analytics", "events") - .await - .expect("committed table lookup should succeed") - .expect("committed table should exist"); - assert_eq!(committed.metadata_location, next_metadata_location); - - drop_table_in_store(&store, "warehouse", &namespace, "events") - .await - .expect("table should drop"); - assert!( - load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events") - .await - .is_err() - ); - } - - #[tokio::test] - async fn register_table_response_adopts_metadata_table_uuid() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let metadata_location = - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - metadata_backend - .put_json( - "warehouse", - metadata_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "metadata-table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - register_table_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: metadata_location.to_string(), - overwrite: false, - }, - true, - ) - .await - .expect("table should register"); - - let entry = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - assert_eq!(entry.table_uuid, "metadata-table-uuid"); - } - - #[tokio::test] - async fn register_table_response_rejects_metadata_without_format_version() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let metadata_location = - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - metadata_backend - .put_json( - "warehouse", - metadata_location, - serde_json::json!({ - "table-uuid": "metadata-table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - assert!( - register_table_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: metadata_location.to_string(), - overwrite: false, - }, - true, - ) - .await - .is_err() - ); - assert!( - store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .is_none() - ); - } - - #[tokio::test] - async fn metadata_location_api_loads_and_updates_current_pointer() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - let entry = table_entry_from_register_request( - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: current_location.to_string(), - overwrite: false, - }, - ) - .expect("table entry should build"); - let table_uuid = entry.table_uuid.clone(); - store.register_table(entry).await.expect("table should register"); - metadata_backend - .put_json( - "warehouse", - current_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": table_uuid, - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") - .await - .expect("metadata location should load"); - assert_eq!( - current.metadata_location, - table_metadata_location_for_client("warehouse", current_location) - ); - let next_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; - metadata_backend - .put_json( - "warehouse", - next_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": table_uuid, - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - let updated = update_table_metadata_location_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - "events", - UpdateTableMetadataLocationRequest { - metadata_location: table_metadata_location_for_client("warehouse", next_location), - version_token: current.version_token.clone(), - commit_id: Some("commit-1".to_string()), - idempotency_key: Some("retry-1".to_string()), - }, - ) - .await - .expect("metadata location should update"); - - assert_eq!(updated.metadata_location, table_metadata_location_for_client("warehouse", next_location)); - assert_eq!(updated.generation, current.generation + 1); - assert_ne!(updated.version_token, current.version_token); - } - - #[tokio::test] - async fn metadata_location_api_rejects_invalid_target_metadata_before_commit() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - store - .register_table( - table_entry_from_register_request( - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: current_location.to_string(), - overwrite: false, - }, - ) - .expect("table entry should build"), - ) - .await - .expect("table should register"); - let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") - .await - .expect("metadata location should load"); - let invalid_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; - metadata_backend - .put_json( - "warehouse", - invalid_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://other-warehouse/tables/table-id" - }), - ) - .await; - - assert!( - update_table_metadata_location_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - "events", - UpdateTableMetadataLocationRequest { - metadata_location: invalid_location.to_string(), - version_token: current.version_token, - commit_id: Some("commit-1".to_string()), - idempotency_key: None, - }, - ) - .await - .is_err() - ); - let unchanged = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") - .await - .expect("metadata location should still load"); - assert_eq!( - unchanged.metadata_location, - table_metadata_location_for_client("warehouse", current_location) - ); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn metadata_location_api_rejects_mismatched_table_uuid_before_commit() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - metadata_backend - .put_json( - "warehouse", - current_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - register_table_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: current_location.to_string(), - overwrite: false, - }, - true, - ) - .await - .expect("table should register"); - let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") - .await - .expect("metadata location should load"); - let mismatched_location = - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; - metadata_backend - .put_json( - "warehouse", - mismatched_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "other-table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - assert!( - update_table_metadata_location_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - "events", - UpdateTableMetadataLocationRequest { - metadata_location: mismatched_location.to_string(), - version_token: current.version_token, - commit_id: Some("commit-1".to_string()), - idempotency_key: None, - }, - ) - .await - .is_err() - ); - let unchanged = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") - .await - .expect("metadata location should still load"); - assert_eq!( - unchanged.metadata_location, - table_metadata_location_for_client("warehouse", current_location) - ); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn catalog_import_and_rollback_use_register_and_commit_paths() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - ensure_table_bucket_entry(&store, bucket, true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - bucket, - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let imported_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - backend - .put_json( - bucket, - &imported_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - let imported = catalog_import_response( - &store, - &backend, - bucket, - &namespace, - "events", - CatalogImportRequest { - metadata_location: table_metadata_location_for_client(bucket, &imported_location), - properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), - }, - true, - ) - .await - .expect("catalog import should register table"); - assert_eq!(imported.metadata_location, table_metadata_location_for_client(bucket, &imported_location)); - let current = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - assert_eq!(current.properties.get("owner").map(String::as_str), Some("lakehouse")); - - let imported_again = catalog_import_response( - &store, - &backend, - bucket, - &namespace, - "events", - CatalogImportRequest { - metadata_location: imported_location.clone(), - properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), - }, - true, - ) - .await - .expect("repeated catalog import should be idempotent"); - assert_eq!( - imported_again.metadata_location, - table_metadata_location_for_client(bucket, &imported_location) - ); - - let rollback_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - backend - .put_json( - bucket, - &rollback_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id", - "last-sequence-number": 2 - }), - ) - .await; - let rollback = rollback_table_response( - &store, - &backend, - bucket, - &namespace, - "events", - RollbackTableRequest { - metadata_location: table_metadata_location_for_client(bucket, &rollback_location), - version_token: current.version_token, - commit_id: Some("rollback-1".to_string()), - idempotency_key: None, - }, - ) - .await - .expect("rollback should commit selected metadata"); - - assert_eq!(rollback.metadata_location, table_metadata_location_for_client(bucket, &rollback_location)); - assert_eq!(rollback.commit_id, "rollback-1"); - } - - #[tokio::test] - async fn rollback_rejects_invalid_target_metadata_before_commit() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - ensure_table_bucket_entry(&store, bucket, true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - bucket, - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - backend - .put_json( - bucket, - ¤t_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - catalog_import_response( - &store, - &backend, - bucket, - &namespace, - "events", - CatalogImportRequest { - metadata_location: current_location.clone(), - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("catalog import should register table"); - let current = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - - let invalid_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - backend - .put_json( - bucket, - &invalid_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://other-warehouse/tables/table-id" - }), - ) - .await; - - assert!( - rollback_table_response( - &store, - &backend, - bucket, - &namespace, - "events", - RollbackTableRequest { - metadata_location: invalid_location, - version_token: current.version_token, - commit_id: Some("rollback-1".to_string()), - idempotency_key: None, - }, - ) - .await - .is_err() - ); - let unchanged = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - - assert_eq!(unchanged.metadata_location, current_location); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn rollback_rejects_mismatched_table_uuid_before_commit() { - let backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); - let bucket = "warehouse"; - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); - ensure_table_bucket_entry(&store, bucket, true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - bucket, - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - backend - .put_json( - bucket, - ¤t_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - catalog_import_response( - &store, - &backend, - bucket, - &namespace, - "events", - CatalogImportRequest { - metadata_location: current_location.clone(), - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("catalog import should register table"); - let current = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - - let mismatched_location = - crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - backend - .put_json( - bucket, - &mismatched_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "other-table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - assert!( - rollback_table_response( - &store, - &backend, - bucket, - &namespace, - "events", - RollbackTableRequest { - metadata_location: mismatched_location, - version_token: current.version_token, - commit_id: Some("rollback-1".to_string()), - idempotency_key: None, - }, - ) - .await - .is_err() - ); - let unchanged = store - .load_table(bucket, "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current_location); - assert_eq!(unchanged.generation, current.generation); - } - - #[tokio::test] - async fn legacy_commit_rejects_mismatched_table_uuid_before_commit() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::default(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - ensure_table_bucket_entry(&store, "warehouse", true) - .await - .expect("table bucket entry should be seeded"); - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["analytics".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("namespace should be created"); - let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; - metadata_backend - .put_json( - "warehouse", - current_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - register_table_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - RegisterTableRequest { - name: "events".to_string(), - metadata_location: current_location.to_string(), - overwrite: false, - }, - true, - ) - .await - .expect("table should register"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let mismatched_location = - ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; - metadata_backend - .put_json( - "warehouse", - mismatched_location, - serde_json::json!({ - "format-version": 2, - "table-uuid": "other-table-uuid", - "location": "s3://warehouse/tables/table-id" - }), - ) - .await; - - assert!( - commit_table_response( - &store, - &metadata_backend, - "warehouse", - &namespace, - "events", - RestCommitTableRequest { - commit_id: Some("commit-1".to_string()), - idempotency_key: None, - operation: Some("append".to_string()), - expected_version_token: Some(current.version_token.clone()), - expected_metadata_location: Some(current.metadata_location.clone()), - new_metadata_location: Some(mismatched_location.to_string()), - requirements: Vec::new(), - updates: Vec::new(), - _identifier: None, - writer: Some("pyiceberg".to_string()), - }, - ) - .await - .is_err() - ); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current_location); - assert_eq!(unchanged.generation, current.generation); - } -} diff --git a/rustfs/src/admin/handlers/table_catalog/config.rs b/rustfs/src/admin/handlers/table_catalog/config.rs new file mode 100644 index 000000000..53d520edf --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/config.rs @@ -0,0 +1,188 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub struct GetCatalogConfigHandler {} + +#[async_trait::async_trait] +impl Operation for GetCatalogConfigHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_table_catalog_request(&req, AdminAction::GetTableCatalogAction).await?; + let warehouse = warehouse_from_config_query(&req.uri)?; + build_json_response(StatusCode::OK, &catalog_config_response(warehouse.as_deref())?) + } +} + +pub struct EnableTableBucketHandler {} + +#[async_trait::async_trait] +impl Operation for EnableTableBucketHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let resource = TableCatalogResource::warehouse(&warehouse); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?; + let store = table_catalog_store()?; + let response = enable_table_bucket_response(&store, &warehouse).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct GetTableBucketHandler {} + +#[async_trait::async_trait] +impl Operation for GetTableBucketHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let resource = TableCatalogResource::warehouse(&warehouse); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableBucketAction).await?; + let store = table_catalog_store()?; + let enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let response = table_bucket_response(&store, &warehouse, enabled).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct GetTableCatalogMigrationHandler {} + +#[async_trait::async_trait] +impl Operation for GetTableCatalogMigrationHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let resource = TableCatalogResource::warehouse(&warehouse); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCatalogAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_object_store()?; + let started = Instant::now(); + let result = store + .plan_durable_strong_backing_migration(&warehouse) + .await + .map_err(catalog_store_error); + record_table_catalog_admin_operation_result("migration", &warehouse, "", "", started, &result); + let response = result?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct MaterializeTableCatalogMigrationHandler {} + +#[async_trait::async_trait] +impl Operation for MaterializeTableCatalogMigrationHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_object_store()?; + let started = Instant::now(); + let result = store + .materialize_durable_strong_backing_migration(&warehouse) + .await + .map_err(catalog_store_error); + record_table_catalog_admin_operation_result("migration-materialize", &warehouse, "", "", started, &result); + let response = result?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct CancelTableCatalogMigrationHandler {} + +#[async_trait::async_trait] +impl Operation for CancelTableCatalogMigrationHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_object_store()?; + let started = Instant::now(); + let result = store + .cancel_durable_strong_backing_migration(&warehouse) + .await + .map_err(catalog_store_error); + record_table_catalog_admin_operation_result("migration-cancel", &warehouse, "", "", started, &result); + let response = result?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct ExternalCatalogBridgeHandler {} + +#[async_trait::async_trait] +impl Operation for ExternalCatalogBridgeHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_object_store()?; + let response = external_catalog_bridge_response(&store, &warehouse, &namespace, &table).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct PutExternalCatalogBridgeHandler {} + +#[async_trait::async_trait] +impl Operation for PutExternalCatalogBridgeHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let store = table_catalog_object_store()?; + let response = put_external_catalog_bridge_response(&store, &warehouse, &namespace, &table, request).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct SyncExternalCatalogBridgeHandler {} + +#[async_trait::async_trait] +impl Operation for SyncExternalCatalogBridgeHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_object_store()?; + if store + .load_table(&warehouse, &namespace.public_name(), &table) + .await + .map_err(catalog_store_error)? + .is_none() + { + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; + } + let request = read_json_body::(req.input).await?; + let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let response = sync_external_catalog_bridge_response( + &store, + &metadata_backend, + &warehouse, + &namespace, + &table, + request, + table_bucket_enabled, + ) + .await?; + build_json_response(StatusCode::OK, &response) + } +} diff --git a/rustfs/src/admin/handlers/table_catalog/credentials.rs b/rustfs/src/admin/handlers/table_catalog/credentials.rs new file mode 100644 index 000000000..2c07f459a --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/credentials.rs @@ -0,0 +1,34 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub struct RestLoadCredentialsHandler {} + +#[async_trait::async_trait] +impl Operation for RestLoadCredentialsHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCredentialsAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let principal = table_catalog_request_principal(&req).await?; + let store = table_catalog_store()?; + let issuer = IamTableCredentialIssuer::from_env(); + let response = load_credentials_response(&store, &warehouse, &namespace, &table, &issuer, Some(&principal)).await?; + build_json_response(StatusCode::OK, &response) + } +} diff --git a/rustfs/src/admin/handlers/table_catalog/maintenance.rs b/rustfs/src/admin/handlers/table_catalog/maintenance.rs new file mode 100644 index 000000000..d576fc7dd --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/maintenance.rs @@ -0,0 +1,223 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub struct RestTableMetadataMaintenanceHandler {} + +#[async_trait::async_trait] +impl Operation for RestTableMetadataMaintenanceHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_object_store()?; + let response = + table_metadata_maintenance_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct GetTableMaintenanceConfigHandler {} + +#[async_trait::async_trait] +impl Operation for GetTableMaintenanceConfigHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let response = store + .get_table_maintenance_config(&warehouse, &namespace.public_name(), &table) + .await + .map_err(catalog_store_error)?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct PutTableMaintenanceConfigHandler {} + +#[async_trait::async_trait] +impl Operation for PutTableMaintenanceConfigHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableLifecycleAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let store = table_catalog_store()?; + let response = store + .put_table_maintenance_config(&warehouse, &namespace.public_name(), &table, request) + .await + .map_err(catalog_store_error)?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct GetTableMaintenanceJobHandler {} + +#[async_trait::async_trait] +impl Operation for GetTableMaintenanceJobHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let job = job_id_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let Some(response) = store + .get_table_metadata_maintenance_report(&warehouse, &namespace.public_name(), &table, &job) + .await + .map_err(catalog_store_error)? + else { + return Err(s3_error!(InvalidRequest, "maintenance job not found")); + }; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct GetTableMaintenanceSchedulerHandler {} + +#[async_trait::async_trait] +impl Operation for GetTableMaintenanceSchedulerHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let response = store + .get_table_maintenance_scheduler_report(&warehouse, &namespace.public_name(), &table) + .await + .map_err(catalog_store_error)?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RunTableMaintenanceWorkerHandler {} + +#[async_trait::async_trait] +impl Operation for RunTableMaintenanceSchedulerHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body_or_default::(req.input).await?; + let store = table_catalog_store()?; + let response = store + .run_table_maintenance_scheduler_once( + &warehouse, + &namespace.public_name(), + &table, + request.scheduler_id().to_string(), + ) + .await + .map_err(catalog_store_error)?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RunTableMaintenanceSchedulerHandler {} + +#[async_trait::async_trait] +impl Operation for RunTableMaintenanceWorkerHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let store = table_catalog_store()?; + let response = store + .run_table_metadata_maintenance_worker_once( + &warehouse, + &namespace.public_name(), + &table, + request.worker_id().to_string(), + ) + .await + .map_err(catalog_store_error)?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct HeartbeatTableMaintenanceJobHandler {} + +#[async_trait::async_trait] +impl Operation for HeartbeatTableMaintenanceJobHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let job = job_id_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let store = table_catalog_store()?; + let response = store + .heartbeat_table_metadata_maintenance_job( + &warehouse, + &namespace.public_name(), + &table, + &job, + &request.lease_id, + &request.worker_id, + ) + .await + .map_err(catalog_store_error)?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct TableMaintenanceQuarantineHandler {} + +#[async_trait::async_trait] +impl Operation for TableMaintenanceQuarantineHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let job = job_id_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let store = table_catalog_store()?; + let response = store + .apply_table_maintenance_quarantine_operation(&warehouse, &namespace.public_name(), &table, &job, request) + .await + .map_err(catalog_store_error)?; + build_json_response(StatusCode::OK, &response) + } +} diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs new file mode 100644 index 000000000..2b43a79e5 --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -0,0 +1,4997 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::runtime_sources::default_admin_usecase; +use crate::admin::runtime_sources::{current_object_store_handle, current_token_signing_key}; +use crate::admin::storage_api::bucket::{metadata::table_catalog_path_hash, metadata_sys}; +use crate::admin::storage_api::runtime::ECStore; +use crate::admin::{ + auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object}, + router::{AdminOperation, Operation, S3Router}, +}; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::{RemoteAddr, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX}; +use crate::table_catalog::{DEFAULT_WAREHOUSE_ID, TableCatalogStore}; +use http::{HeaderMap, HeaderValue, StatusCode}; +use hyper::Method; +use matchit::Params; +use metrics::{counter, histogram}; +use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; +use rustfs_iam::sys::SESSION_POLICY_NAME; +use rustfs_policy::{ + auth::get_new_credentials_with_metadata, + policy::{ + Policy, + action::{Action, AdminAction}, + }, +}; +use rustfs_utils::crypto::{base64_decode_url_safe_no_pad, base64_encode_url_safe_no_pad, hex_sha256}; +use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::num::NonZeroUsize; +use std::time::{Duration as StdDuration, Instant}; +use time::{Duration, OffsetDateTime}; +use uuid::Uuid; + +mod config; +mod credentials; +mod maintenance; +mod namespace; +mod refs; +mod routes; +mod table; +mod view; + +pub use config::*; +pub use credentials::*; +pub use maintenance::*; +pub use namespace::*; +pub use refs::*; +pub use routes::register_table_catalog_route; +pub use table::*; +pub use view::*; + +const JSON_CONTENT_TYPE: &str = "application/json"; +const ENV_TABLE_CATALOG_CREDENTIAL_VENDING: &str = "RUSTFS_TABLE_CATALOG_CREDENTIAL_VENDING"; +const ENV_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: &str = "RUSTFS_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS"; +const DEFAULT_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 15 * 60; +const MIN_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60; +const MAX_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60 * 60; +const WAREHOUSE_PROPERTY: &str = "warehouse"; +const PREFIX_PROPERTY: &str = "prefix"; +const ICEBERG_ERROR_ALREADY_EXISTS: &str = "AlreadyExistsException"; +const ICEBERG_ERROR_BAD_REQUEST: &str = "BadRequestException"; +const ICEBERG_ERROR_COMMIT_FAILED: &str = "CommitFailedException"; +const ICEBERG_ERROR_NAMESPACE_NOT_EMPTY: &str = "NamespaceNotEmptyException"; +const ICEBERG_ERROR_NO_SUCH_NAMESPACE: &str = "NoSuchNamespaceException"; +const ICEBERG_ERROR_NO_SUCH_RESOURCE: &str = "NoSuchResourceException"; +const ICEBERG_ERROR_NO_SUCH_TABLE: &str = "NoSuchTableException"; +const ICEBERG_ERROR_NO_SUCH_VIEW: &str = "NoSuchViewException"; +const ICEBERG_ERROR_REST: &str = "RESTException"; +const REST_PAGE_TOKEN_VERSION: u8 = 1; +const REST_PAGE_TOKEN_MAX_LENGTH: usize = 16 * 1024; +const REST_DEFAULT_PAGE_SIZE: usize = 1000; +const REST_MAX_PAGE_SIZE: usize = 1000; +const REST_PAGE_TOKEN_QUERY_PARAMETER: &str = "pageToken"; +const REST_PAGE_SIZE_QUERY_PARAMETER: &str = "pageSize"; +const CATALOG_ENDPOINT_PREFIX_CONFIG_KEY: &str = "rustfs.catalog-endpoint-prefix"; +const CATALOG_COMPAT_ENDPOINT_PREFIX_CONFIG_KEY: &str = "rustfs.catalog-compat-endpoint-prefix"; +const CATALOG_BACKING_CONFIG_KEY: &str = "rustfs.catalog-backing"; +const CREDENTIAL_VENDING_CONFIG_KEY: &str = "rustfs.credential-vending"; +const CREDENTIAL_VENDING_REASON_CONFIG_KEY: &str = "rustfs.credential-vending-reason"; +const CREDENTIAL_SCOPE_CONFIG_KEY: &str = "rustfs.credential-scope"; +const CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY: &str = "rustfs.credential-scope-prefix"; +const CREDENTIAL_MODE_CONFIG_KEY: &str = "rustfs.credential-mode"; +const CREDENTIAL_EXPIRATION_CONFIG_KEY: &str = "rustfs.credential-expiration-unix-seconds"; +const CREDENTIAL_VENDING_UNSUPPORTED: &str = "unsupported"; +const CREDENTIAL_VENDING_SUPPORTED: &str = "supported"; +const CREDENTIAL_VENDING_UNSUPPORTED_REASON: &str = "temporary-credentials-not-implemented"; +const CREDENTIAL_VENDING_DISABLED_REASON: &str = "credential-vending-disabled"; +const CREDENTIAL_SCOPE_WAREHOUSE_PREFIX: &str = "warehouse-prefix"; +const CREDENTIAL_SCOPE_TABLE_PREFIX: &str = "table-prefix"; +const CREDENTIAL_MODE_CLIENT_PROVIDED: &str = "client-provided-s3-credentials-required"; +const CREDENTIAL_MODE_CATALOG_VENDED: &str = "catalog-vended-temporary-credentials"; +const S3_ACCESS_KEY_ID_CONFIG_KEY: &str = "s3.access-key-id"; +const S3_SECRET_ACCESS_KEY_CONFIG_KEY: &str = "s3.secret-access-key"; +const S3_SESSION_TOKEN_CONFIG_KEY: &str = "s3.session-token"; +const TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT: &str = "namespaces"; +const TABLE_CATALOG_TABLE_RESOURCE_ROOT: &str = "tables"; +const TABLE_CATALOG_VIEW_RESOURCE_ROOT: &str = "views"; +const TABLE_CATALOG_ADMIN_OPERATION_SLOW_LOG_THRESHOLD: StdDuration = StdDuration::from_secs(2); +const DEFAULT_TABLE_MAINTENANCE_SCHEDULER_ID: &str = "rustfs-maintenance-scheduler"; +const DEFAULT_TABLE_MAINTENANCE_WORKER_ID: &str = "rustfs-maintenance-worker"; +const EXTERNAL_CATALOG_BRIDGE_STATUS_UNCONFIGURED: &str = "bridge-unconfigured"; +const EXTERNAL_CATALOG_BRIDGE_STATUS_CONFIGURED: &str = "bridge-configured"; +const EXTERNAL_CATALOG_BRIDGE_SYNC_STATUS: &str = "synced"; +const EXTERNAL_CATALOG_BRIDGE_SUPPORTED_STATUS: &str = "operator-sync-supported"; +const EXTERNAL_CATALOG_BRIDGE_SUPPORTED_REASON: &str = + "operator-supplied metadata pointer sync is supported; online vendor SDK synchronization is not claimed"; +const EXTERNAL_CATALOG_POLICY_MODE_RUSTFS: &str = "rustfs-authoritative"; +const EXTERNAL_CATALOG_CREDENTIAL_MODE_RUSTFS: &str = "rustfs-table-credentials"; +const EXTERNAL_CATALOG_SYNC_MODE_MANUAL: &str = "manual"; +const EXTERNAL_CATALOG_ROLLBACK_RETAIN_CURRENT: &str = "retain-current-pointer"; +const EXTERNAL_CATALOG_SYNC_OPERATION: &str = "external-catalog-sync"; +const EXTERNAL_CATALOG_SYNC_WRITER: &str = "rustfs-external-catalog-bridge"; +const EXTERNAL_CATALOG_ACTION_REGISTERED: &str = "registered"; +const EXTERNAL_CATALOG_ACTION_COMMITTED: &str = "committed"; +const EXTERNAL_CATALOG_BRIDGE_CAPABILITIES: &[&str] = &["polaris", "glue", "dlf", "hive-metastore"]; +const TABLE_CATALOG_ENDPOINTS: &[&str] = &[ + "GET /v1/{prefix}/namespaces", + "POST /v1/{prefix}/namespaces", + "GET /v1/{prefix}/namespaces/{namespace}", + "HEAD /v1/{prefix}/namespaces/{namespace}", + "DELETE /v1/{prefix}/namespaces/{namespace}", + "GET /v1/{prefix}/namespaces/{namespace}/tables", + "POST /v1/{prefix}/namespaces/{namespace}/tables", + "POST /v1/{prefix}/namespaces/{namespace}/register", + "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials", + "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "PUT /buckets/{warehouse}", + "GET /buckets/{warehouse}", + "GET /{warehouse}/catalog/migration", + "POST /{warehouse}/catalog/migration", + "DELETE /{warehouse}/catalog/migration", + "GET /{warehouse}/namespaces", + "POST /{warehouse}/namespaces", + "GET /{warehouse}/namespaces/{namespace}", + "HEAD /{warehouse}/namespaces/{namespace}", + "DELETE /{warehouse}/namespaces/{namespace}", + "GET /{warehouse}/namespaces/{namespace}/tables", + "POST /{warehouse}/namespaces/{namespace}/tables", + "POST /{warehouse}/namespaces/{namespace}/register", + "GET /{warehouse}/namespaces/{namespace}/views", + "POST /{warehouse}/namespaces/{namespace}/views", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}", + "HEAD /{warehouse}/namespaces/{namespace}/tables/{table}", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/credentials", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}", + "DELETE /{warehouse}/namespaces/{namespace}/tables/{table}", + "GET /{warehouse}/namespaces/{namespace}/views/{view}", + "HEAD /{warehouse}/namespaces/{namespace}/views/{view}", + "POST /{warehouse}/namespaces/{namespace}/views/{view}", + "DELETE /{warehouse}/namespaces/{namespace}/views/{view}", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/refs", + "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}", + "DELETE /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location", + "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config", + "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/heartbeat", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/quarantine", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external", + "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external/sync", + "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/recovery", + "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback", +]; + +static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {}; +static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {}; +static GET_TABLE_BUCKET_HANDLER: GetTableBucketHandler = GetTableBucketHandler {}; +static GET_TABLE_CATALOG_MIGRATION_HANDLER: GetTableCatalogMigrationHandler = GetTableCatalogMigrationHandler {}; +static MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER: MaterializeTableCatalogMigrationHandler = + MaterializeTableCatalogMigrationHandler {}; +static CANCEL_TABLE_CATALOG_MIGRATION_HANDLER: CancelTableCatalogMigrationHandler = CancelTableCatalogMigrationHandler {}; +static LIST_NAMESPACES_HANDLER: RestListNamespacesHandler = RestListNamespacesHandler {}; +static CREATE_NAMESPACE_HANDLER: RestCreateNamespaceHandler = RestCreateNamespaceHandler {}; +static GET_NAMESPACE_HANDLER: RestGetNamespaceHandler = RestGetNamespaceHandler {}; +static NAMESPACE_EXISTS_HANDLER: RestNamespaceExistsHandler = RestNamespaceExistsHandler {}; +static DROP_NAMESPACE_HANDLER: RestDropNamespaceHandler = RestDropNamespaceHandler {}; +static LIST_TABLES_HANDLER: RestListTablesHandler = RestListTablesHandler {}; +static CREATE_TABLE_HANDLER: RestCreateTableHandler = RestCreateTableHandler {}; +static REGISTER_TABLE_HANDLER: RestRegisterTableHandler = RestRegisterTableHandler {}; +static LIST_VIEWS_HANDLER: RestListViewsHandler = RestListViewsHandler {}; +static CREATE_VIEW_HANDLER: RestCreateViewHandler = RestCreateViewHandler {}; +static LOAD_TABLE_HANDLER: RestLoadTableHandler = RestLoadTableHandler {}; +static TABLE_EXISTS_HANDLER: RestTableExistsHandler = RestTableExistsHandler {}; +static LOAD_CREDENTIALS_HANDLER: RestLoadCredentialsHandler = RestLoadCredentialsHandler {}; +static COMMIT_TABLE_HANDLER: RestCommitTableHandler = RestCommitTableHandler {}; +static DROP_TABLE_HANDLER: RestDropTableHandler = RestDropTableHandler {}; +static LOAD_VIEW_HANDLER: RestLoadViewHandler = RestLoadViewHandler {}; +static VIEW_EXISTS_HANDLER: RestViewExistsHandler = RestViewExistsHandler {}; +static REPLACE_VIEW_HANDLER: RestReplaceViewHandler = RestReplaceViewHandler {}; +static DROP_VIEW_HANDLER: RestDropViewHandler = RestDropViewHandler {}; +static LIST_TABLE_REFS_HANDLER: ListTableRefsHandler = ListTableRefsHandler {}; +static PUT_TABLE_REF_HANDLER: PutTableRefHandler = PutTableRefHandler {}; +static DELETE_TABLE_REF_HANDLER: DeleteTableRefHandler = DeleteTableRefHandler {}; +static GET_TABLE_METADATA_LOCATION_HANDLER: GetTableMetadataLocationHandler = GetTableMetadataLocationHandler {}; +static UPDATE_TABLE_METADATA_LOCATION_HANDLER: UpdateTableMetadataLocationHandler = UpdateTableMetadataLocationHandler {}; +static TABLE_METADATA_MAINTENANCE_HANDLER: RestTableMetadataMaintenanceHandler = RestTableMetadataMaintenanceHandler {}; +static GET_TABLE_MAINTENANCE_CONFIG_HANDLER: GetTableMaintenanceConfigHandler = GetTableMaintenanceConfigHandler {}; +static PUT_TABLE_MAINTENANCE_CONFIG_HANDLER: PutTableMaintenanceConfigHandler = PutTableMaintenanceConfigHandler {}; +static GET_TABLE_MAINTENANCE_JOB_HANDLER: GetTableMaintenanceJobHandler = GetTableMaintenanceJobHandler {}; +static GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER: GetTableMaintenanceSchedulerHandler = GetTableMaintenanceSchedulerHandler {}; +static RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER: RunTableMaintenanceSchedulerHandler = RunTableMaintenanceSchedulerHandler {}; +static RUN_TABLE_MAINTENANCE_WORKER_HANDLER: RunTableMaintenanceWorkerHandler = RunTableMaintenanceWorkerHandler {}; +static HEARTBEAT_TABLE_MAINTENANCE_JOB_HANDLER: HeartbeatTableMaintenanceJobHandler = HeartbeatTableMaintenanceJobHandler {}; +static TABLE_MAINTENANCE_QUARANTINE_HANDLER: TableMaintenanceQuarantineHandler = TableMaintenanceQuarantineHandler {}; +static EXPORT_TABLE_CATALOG_HANDLER: ExportTableCatalogHandler = ExportTableCatalogHandler {}; +static IMPORT_TABLE_CATALOG_HANDLER: ImportTableCatalogHandler = ImportTableCatalogHandler {}; +static EXTERNAL_CATALOG_BRIDGE_HANDLER: ExternalCatalogBridgeHandler = ExternalCatalogBridgeHandler {}; +static PUT_EXTERNAL_CATALOG_BRIDGE_HANDLER: PutExternalCatalogBridgeHandler = PutExternalCatalogBridgeHandler {}; +static SYNC_EXTERNAL_CATALOG_BRIDGE_HANDLER: SyncExternalCatalogBridgeHandler = SyncExternalCatalogBridgeHandler {}; +static GET_TABLE_CATALOG_DIAGNOSTICS_HANDLER: GetTableCatalogDiagnosticsHandler = GetTableCatalogDiagnosticsHandler {}; +static RECOVER_TABLE_CATALOG_HANDLER: RecoverTableCatalogHandler = RecoverTableCatalogHandler {}; +static ROLLBACK_TABLE_CATALOG_HANDLER: RollbackTableCatalogHandler = RollbackTableCatalogHandler {}; + +#[derive(Debug, Serialize)] +struct CatalogConfigResponse { + defaults: BTreeMap, + overrides: BTreeMap, + endpoints: Vec<&'static str>, + admin_discovery: CatalogAdminDiscovery, +} + +#[derive(Debug, Serialize)] +struct CatalogAdminDiscovery { + #[serde(rename = "runtimeCapabilities")] + runtime_capabilities: &'static str, + #[serde(rename = "clusterSnapshot")] + cluster_snapshot: &'static str, + #[serde(rename = "extensionsCatalog")] + extensions_catalog: &'static str, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CreateNamespaceRequest { + namespace: Vec, + #[serde(default)] + properties: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RegisterTableRequest { + name: String, + #[serde(rename = "metadata-location")] + metadata_location: String, + #[serde(default)] + overwrite: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CreateTableRequest { + name: String, + #[serde(default)] + location: Option, + schema: serde_json::Value, + #[serde(default, rename = "partition-spec")] + partition_spec: Option, + #[serde(default, rename = "write-order")] + write_order: Option, + #[serde(default, rename = "stage-create")] + stage_create: bool, + #[serde(default)] + properties: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CreateViewRequest { + name: String, + #[serde(default)] + location: Option, + schema: serde_json::Value, + #[serde(rename = "view-version")] + view_version: serde_json::Value, + #[serde(default)] + properties: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RestCommitTableRequest { + #[serde(default, rename = "identifier")] + _identifier: Option, + #[serde(default, rename = "commit-id")] + commit_id: Option, + #[serde(default, rename = "idempotency-key")] + idempotency_key: Option, + #[serde(default)] + operation: Option, + #[serde(default, rename = "expected-version-token")] + expected_version_token: Option, + #[serde(default, rename = "expected-metadata-location")] + expected_metadata_location: Option, + #[serde(default, rename = "new-metadata-location")] + new_metadata_location: Option, + #[serde(default)] + requirements: Vec, + #[serde(default)] + updates: Vec, + #[serde(default)] + writer: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RestCommitViewRequest { + #[serde(default, rename = "commit-id")] + commit_id: Option, + #[serde(default, rename = "expected-version-token")] + expected_version_token: Option, + #[serde(default, rename = "expected-metadata-location")] + expected_metadata_location: Option, + #[serde(default, rename = "new-metadata-location")] + new_metadata_location: Option, + #[serde(default)] + requirements: Vec, + #[serde(default)] + updates: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PutTableRefRequest { + #[serde(rename = "snapshot-id")] + snapshot_id: i64, + #[serde(rename = "type")] + ref_type: String, + #[serde(default, rename = "expected-snapshot-id")] + expected_snapshot_id: Option, + #[serde(default, rename = "min-snapshots-to-keep")] + min_snapshots_to_keep: Option, + #[serde(default, rename = "max-snapshot-age-ms")] + max_snapshot_age_ms: Option, + #[serde(default, rename = "max-ref-age-ms")] + max_ref_age_ms: Option, + #[serde(default, rename = "commit-id")] + commit_id: Option, + #[serde(default, rename = "idempotency-key")] + idempotency_key: Option, + #[serde(default)] + writer: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct DeleteTableRefRequest { + #[serde(default, rename = "expected-snapshot-id")] + expected_snapshot_id: Option, + #[serde(default)] + force: bool, + #[serde(default, rename = "commit-id")] + commit_id: Option, + #[serde(default, rename = "idempotency-key")] + idempotency_key: Option, + #[serde(default)] + writer: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct TableMetadataMaintenanceRequest { + #[serde(default, rename = "retain-recent-metadata-files")] + retain_recent_metadata_files: usize, + #[serde(default)] + delete: bool, + #[serde(default, rename = "snapshot-expiration")] + snapshot_expiration: Option, + #[serde(default, rename = "commit-snapshot-expiration")] + commit_snapshot_expiration: bool, + #[serde(default)] + compaction: Option, + #[serde(default, rename = "commit-compaction")] + commit_compaction: bool, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct TableMaintenanceSchedulerRunRequest { + #[serde(default, rename = "scheduler-id")] + scheduler_id: Option, +} + +impl TableMaintenanceSchedulerRunRequest { + fn scheduler_id(&self) -> &str { + self.scheduler_id.as_deref().unwrap_or(DEFAULT_TABLE_MAINTENANCE_SCHEDULER_ID) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct TableMaintenanceWorkerRunRequest { + #[serde(default, rename = "worker-id")] + worker_id: Option, +} + +impl TableMaintenanceWorkerRunRequest { + fn worker_id(&self) -> &str { + self.worker_id.as_deref().unwrap_or(DEFAULT_TABLE_MAINTENANCE_WORKER_ID) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct TableMaintenanceHeartbeatRequest { + #[serde(rename = "lease-id")] + lease_id: String, + #[serde(rename = "worker-id")] + worker_id: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct UpdateTableMetadataLocationRequest { + #[serde(rename = "metadata-location", alias = "metadataLocation")] + metadata_location: String, + #[serde(rename = "version-token", alias = "versionToken")] + version_token: String, + #[serde(default, rename = "commit-id", alias = "commitId")] + commit_id: Option, + #[serde(default, rename = "idempotency-key", alias = "idempotencyKey")] + idempotency_key: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CatalogImportRequest { + #[serde(rename = "metadata-location", alias = "metadataLocation")] + metadata_location: String, + #[serde(default)] + properties: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExternalCatalogBridgeRequest { + catalog: String, + #[serde(default, rename = "external-catalog-id", alias = "externalCatalogId")] + external_catalog_id: Option, + #[serde(rename = "external-namespace", alias = "externalNamespace")] + external_namespace: String, + #[serde(rename = "external-table", alias = "externalTable")] + external_table: String, + #[serde(default, rename = "external-table-uuid", alias = "externalTableUuid")] + external_table_uuid: Option, + #[serde(default, rename = "metadata-location", alias = "metadataLocation")] + metadata_location: Option, + #[serde(default, rename = "external-version-token", alias = "externalVersionToken")] + external_version_token: Option, + #[serde(default, rename = "policy-mode", alias = "policyMode")] + policy_mode: Option, + #[serde(default, rename = "credential-mode", alias = "credentialMode")] + credential_mode: Option, + #[serde(default, rename = "sync-mode", alias = "syncMode")] + sync_mode: Option, + #[serde(default)] + properties: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExternalCatalogBridgeSyncRequest { + catalog: String, + #[serde(default, rename = "external-catalog-id", alias = "externalCatalogId")] + external_catalog_id: Option, + #[serde(rename = "external-namespace", alias = "externalNamespace")] + external_namespace: String, + #[serde(rename = "external-table", alias = "externalTable")] + external_table: String, + #[serde(default, rename = "external-table-uuid", alias = "externalTableUuid")] + external_table_uuid: Option, + #[serde(rename = "metadata-location", alias = "metadataLocation")] + metadata_location: String, + #[serde(default, rename = "external-version-token", alias = "externalVersionToken")] + external_version_token: Option, + #[serde(default, rename = "expected-version-token", alias = "expectedVersionToken")] + expected_version_token: Option, + #[serde(default, rename = "expected-metadata-location", alias = "expectedMetadataLocation")] + expected_metadata_location: Option, + #[serde(default, rename = "commit-id", alias = "commitId")] + commit_id: Option, + #[serde(default, rename = "idempotency-key", alias = "idempotencyKey")] + idempotency_key: Option, + #[serde(default, rename = "policy-mode", alias = "policyMode")] + policy_mode: Option, + #[serde(default, rename = "credential-mode", alias = "credentialMode")] + credential_mode: Option, + #[serde(default, rename = "rollback-strategy", alias = "rollbackStrategy")] + rollback_strategy: Option, + #[serde(default)] + properties: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RollbackTableRequest { + #[serde(rename = "metadata-location", alias = "metadataLocation")] + metadata_location: String, + #[serde(rename = "version-token", alias = "versionToken")] + version_token: String, + #[serde(default, rename = "commit-id", alias = "commitId")] + commit_id: Option, + #[serde(default, rename = "idempotency-key", alias = "idempotencyKey")] + idempotency_key: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "kebab-case")] +struct TableRefsResponse { + table_bucket: String, + namespace: String, + table: String, + current_metadata_location: String, + current_snapshot_id: Option, + protected_ref_count: usize, + user_defined_ref_count: usize, + refs: BTreeMap, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "kebab-case")] +struct ExternalCatalogBridgeResponse { + table_bucket: String, + namespace: String, + table: String, + status: String, + supported_import: String, + #[serde(default)] + capabilities: Vec, + #[serde(default)] + unsupported_bridges: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + bridge: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "kebab-case")] +struct ExternalCatalogBridgeCapability { + catalog: String, + status: String, + reason: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "kebab-case")] +struct ExternalCatalogBridgeStateResponse { + catalog: String, + external_catalog_id: Option, + external_namespace: String, + external_table: String, + external_table_uuid: Option, + metadata_location: Option, + external_version_token: Option, + policy_mode: String, + credential_mode: String, + sync_mode: String, + rollback_strategy: String, + last_sync_status: Option, + last_synced_metadata_location: Option, + properties: BTreeMap, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "kebab-case")] +struct ExternalCatalogBridgeSyncResponse { + action: String, + table: RestLoadTableResponse, + bridge: ExternalCatalogBridgeResponse, +} + +#[derive(Debug, Serialize)] +struct TableBucketResponse { + #[serde(rename = "table-bucket")] + table_bucket: String, + enabled: bool, + #[serde(rename = "catalog-type")] + catalog_type: String, + warehouse: String, + #[serde(rename = "warehouse-location")] + warehouse_location: String, + #[serde(rename = "catalog-uri")] + catalog_uri: String, + #[serde(rename = "compat-catalog-uri")] + compat_catalog_uri: String, + #[serde(rename = "credential-vending")] + credential_vending: &'static str, + #[serde(rename = "credential-scope")] + credential_scope: &'static str, + #[serde(rename = "credential-scope-prefix")] + credential_scope_prefix: String, + #[serde(rename = "catalog-entry-present")] + catalog_entry_present: bool, + properties: BTreeMap, +} + +#[derive(Debug, Serialize)] +struct RestNamespaceResponse { + namespace: Vec, + properties: BTreeMap, +} + +#[derive(Debug, Serialize)] +struct RestListNamespacesResponse { + namespaces: Vec>, + #[serde(rename = "next-page-token")] + next_page_token: Option, +} + +#[derive(Debug, Serialize)] +struct RestTableIdentifier { + namespace: Vec, + name: String, +} + +#[derive(Debug, Serialize)] +struct RestListTablesResponse { + identifiers: Vec, + #[serde(rename = "next-page-token")] + next_page_token: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RestPageToken { + version: u8, + context: String, + cursor: String, +} + +#[derive(Debug)] +enum RestPagination { + Unpaginated, + Paginated { + cursor: Option, + limit: NonZeroUsize, + context: String, + }, +} + +impl RestPagination { + fn page_request(&self) -> Option<(Option<&str>, NonZeroUsize)> { + match self { + Self::Unpaginated => None, + Self::Paginated { cursor, limit, .. } => Some((cursor.as_deref(), *limit)), + } + } + + fn next_page_token(&self, cursor: Option) -> S3Result> { + match (self, cursor) { + (Self::Unpaginated, _) | (_, None) => Ok(None), + (Self::Paginated { context, .. }, Some(cursor)) => encode_rest_page_token(&cursor, context).map(Some), + } + } +} + +#[derive(Clone, Copy)] +struct RestPageContext<'a> { + resource: &'static str, + warehouse: &'a str, + namespace: Option<&'a str>, +} + +#[derive(Debug, Serialize)] +struct RestListViewsResponse { + identifiers: Vec, + #[serde(rename = "next-page-token")] + next_page_token: Option, +} + +#[derive(Debug, Serialize)] +struct RestLoadViewResponse { + #[serde(rename = "metadata-location")] + metadata_location: String, + metadata: serde_json::Value, + config: BTreeMap, +} + +#[derive(Debug, Serialize)] +struct RestStorageCredential { + prefix: String, + config: BTreeMap, +} + +#[derive(Debug, Clone)] +struct TableCredentialScope { + scope_prefix: String, + object_prefix: String, +} + +#[derive(Debug, Clone)] +struct TableCredentialIssueRequest<'a> { + entry: &'a crate::table_catalog::TableEntry, + principal: Option<&'a rustfs_credentials::Credentials>, + scope_prefix: String, + object_prefix: String, +} + +#[derive(Debug, Clone)] +struct IssuedTableCredentials { + access_key_id: String, + secret_access_key: String, + session_token: String, + expiration: OffsetDateTime, +} + +#[async_trait::async_trait] +trait TableCredentialIssuer: Sync { + fn enabled(&self) -> bool { + true + } + + async fn issue_table_credentials(&self, request: TableCredentialIssueRequest<'_>) + -> S3Result>; +} + +#[cfg(test)] +struct DisabledTableCredentialIssuer; + +#[cfg(test)] +#[async_trait::async_trait] +impl TableCredentialIssuer for DisabledTableCredentialIssuer { + fn enabled(&self) -> bool { + false + } + + async fn issue_table_credentials( + &self, + _request: TableCredentialIssueRequest<'_>, + ) -> S3Result> { + Ok(None) + } +} + +struct IamTableCredentialIssuer { + enabled: bool, + ttl_seconds: i64, +} + +impl IamTableCredentialIssuer { + fn from_env() -> Self { + Self { + enabled: table_credential_vending_enabled(), + ttl_seconds: table_credential_ttl_seconds(), + } + } +} + +#[async_trait::async_trait] +impl TableCredentialIssuer for IamTableCredentialIssuer { + fn enabled(&self) -> bool { + self.enabled + } + + async fn issue_table_credentials( + &self, + request: TableCredentialIssueRequest<'_>, + ) -> S3Result> { + if !self.enabled { + return Ok(None); + } + + let Some(principal) = request.principal else { + return Err(s3_error!(InvalidRequest, "authentication required for table credentials")); + }; + if principal.is_temp() || principal.is_service_account() { + return Err(s3_error!( + AccessDenied, + "table credential vending does not allow chained temporary credentials" + )); + } + + let policy = table_credential_session_policy(request.entry, &request.object_prefix)?; + let policy_buf = serde_json::to_vec(&policy) + .map_err(|err| s3_error!(InternalError, "failed to serialize table credential session policy: {}", err))?; + let expiration = OffsetDateTime::now_utc().saturating_add(Duration::seconds(self.ttl_seconds)); + let mut claims: HashMap = principal.claims.clone().unwrap_or_default(); + claims.insert( + "exp".to_string(), + serde_json::Value::Number(serde_json::Number::from(expiration.unix_timestamp())), + ); + claims.insert("parent".to_string(), serde_json::Value::String(principal.access_key.clone())); + claims.insert( + SESSION_POLICY_NAME.to_string(), + serde_json::Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&policy_buf)), + ); + claims.insert( + "rustfs:table-bucket".to_string(), + serde_json::Value::String(request.entry.table_bucket.clone()), + ); + claims.insert("rustfs:table-id".to_string(), serde_json::Value::String(request.entry.table_id.clone())); + claims.insert( + "rustfs:credential-scope-prefix".to_string(), + serde_json::Value::String(request.scope_prefix.clone()), + ); + + let secret = current_token_signing_key().ok_or_else(|| s3_error!(InternalError, "token signing key not initialized"))?; + let mut credential = get_new_credentials_with_metadata(&claims, &secret) + .map_err(|err| s3_error!(InternalError, "failed to generate table credentials: {}", err))?; + bind_table_credential_parent(&mut credential, principal); + + let iam_store = + crate::admin::runtime_sources::current_ready_iam_handle().map_err(|_| s3_error!(InternalError, "iam not init"))?; + iam_store + .set_temp_user(&credential.access_key, &credential, None) + .await + .map_err(|_| s3_error!(InternalError, "failed to store table credentials"))?; + + Ok(Some(IssuedTableCredentials { + access_key_id: credential.access_key, + secret_access_key: credential.secret_key, + session_token: credential.session_token, + expiration, + })) + } +} + +fn bind_table_credential_parent(credential: &mut rustfs_credentials::Credentials, principal: &rustfs_credentials::Credentials) { + credential.parent_user = principal.access_key.clone(); +} + +#[derive(Debug, Serialize)] +struct RestLoadTableResponse { + #[serde(rename = "metadata-location")] + metadata_location: String, + metadata: serde_json::Value, + config: BTreeMap, + #[serde(rename = "storage-credentials")] + storage_credentials: Vec, +} + +#[derive(Debug, Serialize)] +struct RestLoadCredentialsResponse { + config: BTreeMap, + #[serde(rename = "storage-credentials")] + storage_credentials: Vec, +} + +#[derive(Debug, Serialize)] +struct RestCommitTableResponse { + #[serde(rename = "metadata-location")] + metadata_location: String, + metadata: serde_json::Value, + #[serde(rename = "version-token")] + version_token: String, + generation: u64, + #[serde(rename = "commit-id")] + commit_id: String, +} + +#[derive(Debug, Serialize)] +struct TableMetadataLocationResponse { + #[serde(rename = "metadata-location")] + metadata_location: String, + #[serde(rename = "version-token")] + version_token: String, + generation: u64, + #[serde(rename = "warehouse-location")] + warehouse_location: String, +} + +fn catalog_config_response(warehouse: Option<&str>) -> S3Result { + let usecase = default_admin_usecase(); + let backing_mode = crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)?; + let mut overrides = BTreeMap::new(); + if backing_mode != crate::table_catalog::TableCatalogBackingMode::ObjectBacked { + overrides.insert(CATALOG_BACKING_CONFIG_KEY.to_string(), backing_mode.as_str().to_string()); + } + let mut defaults = BTreeMap::from([ + (WAREHOUSE_PROPERTY.to_string(), DEFAULT_WAREHOUSE_ID.to_string()), + (CATALOG_ENDPOINT_PREFIX_CONFIG_KEY.to_string(), TABLE_CATALOG_PREFIX.to_string()), + ( + CATALOG_COMPAT_ENDPOINT_PREFIX_CONFIG_KEY.to_string(), + TABLE_CATALOG_COMPAT_PREFIX.to_string(), + ), + ( + CATALOG_BACKING_CONFIG_KEY.to_string(), + crate::table_catalog::TABLE_CATALOG_BACKING_OBJECT.to_string(), + ), + ]); + if let Some(warehouse) = warehouse { + defaults.insert(PREFIX_PROPERTY.to_string(), warehouse.to_string()); + } + Ok(CatalogConfigResponse { + defaults, + overrides, + endpoints: TABLE_CATALOG_ENDPOINTS.to_vec(), + admin_discovery: CatalogAdminDiscovery { + runtime_capabilities: usecase.runtime_capabilities_route(), + cluster_snapshot: usecase.cluster_snapshot_route(), + extensions_catalog: usecase.extensions_catalog_route(), + }, + }) +} + +fn build_json_response(status: StatusCode, body: &T) -> S3Result> { + let data = serde_json::to_vec(body).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE)); + Ok(S3Response::with_headers((status, Body::from(data)), headers)) +} + +fn empty_response(status: StatusCode) -> S3Response<(StatusCode, Body)> { + S3Response::new((status, Body::default())) +} + +fn duration_millis_u64(duration: StdDuration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +fn table_catalog_admin_operation_result_label(result: &Result) -> &'static str { + if result.is_ok() { "success" } else { "failure" } +} + +fn record_table_catalog_admin_operation_result( + operation: &str, + warehouse: &str, + namespace: &str, + table: &str, + started: Instant, + result: &Result, +) { + let elapsed = started.elapsed(); + let result_label = table_catalog_admin_operation_result_label(result); + counter!( + "rustfs_table_catalog_admin_operations_total", + "operation" => operation.to_string(), + "result" => result_label.to_string() + ) + .increment(1); + histogram!( + "rustfs_table_catalog_admin_operation_duration_seconds", + "operation" => operation.to_string(), + "result" => result_label.to_string() + ) + .record(elapsed.as_secs_f64()); + + if result.is_err() { + tracing::warn!( + operation, + warehouse, + namespace, + table, + result = result_label, + duration_ms = duration_millis_u64(elapsed), + "table catalog admin operation failed" + ); + } else if elapsed >= TABLE_CATALOG_ADMIN_OPERATION_SLOW_LOG_THRESHOLD { + tracing::warn!( + operation, + warehouse, + namespace, + table, + duration_ms = duration_millis_u64(elapsed), + "slow table catalog admin operation" + ); + } +} + +fn exists_status(exists: bool) -> StatusCode { + if exists { + StatusCode::NO_CONTENT + } else { + StatusCode::NOT_FOUND + } +} + +async fn authorize_table_catalog_request(req: &S3Request, action: AdminAction) -> S3Result<()> { + let Some(input_cred) = &req.credentials else { + return Err(s3_error!(InvalidRequest, "authentication required")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + vec![Action::AdminAction(action)], + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ) + .await +} + +#[derive(Debug, Clone)] +struct TableCatalogResource<'a> { + warehouse: &'a str, + namespace: Option, + table: Option, + view: Option, +} + +impl<'a> TableCatalogResource<'a> { + fn warehouse(warehouse: &'a str) -> Self { + Self { + warehouse, + namespace: None, + table: None, + view: None, + } + } + + fn namespace(warehouse: &'a str, namespace: &crate::table_catalog::Namespace) -> Self { + Self { + warehouse, + namespace: Some(namespace.storage_id()), + table: None, + view: None, + } + } + + fn table(warehouse: &'a str, namespace: &crate::table_catalog::Namespace, table: &str) -> Self { + Self { + warehouse, + namespace: Some(namespace.storage_id()), + table: Some(table.to_string()), + view: None, + } + } + + fn view(warehouse: &'a str, namespace: &crate::table_catalog::Namespace, view: &str) -> Self { + Self { + warehouse, + namespace: Some(namespace.storage_id()), + table: None, + view: Some(view.to_string()), + } + } + + fn object_path(&self) -> Option { + match (&self.namespace, &self.table, &self.view) { + (Some(namespace), Some(table), None) => Some(format!( + "{TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT}/{namespace}/{TABLE_CATALOG_TABLE_RESOURCE_ROOT}/{table}" + )), + (Some(namespace), None, Some(view)) => Some(format!( + "{TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT}/{namespace}/{TABLE_CATALOG_VIEW_RESOURCE_ROOT}/{view}" + )), + (Some(namespace), None, None) => Some(format!("{TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT}/{namespace}")), + _ => None, + } + } +} + +async fn authorize_table_catalog_resource_request( + req: &S3Request, + resource: &TableCatalogResource<'_>, + action: AdminAction, +) -> S3Result<()> { + let Some(input_cred) = &req.credentials else { + return Err(s3_error!(InvalidRequest, "authentication required")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let object_path = resource.object_path(); + validate_admin_request_with_bucket_object( + &req.headers, + &cred, + owner, + false, + vec![Action::AdminAction(action)], + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + AdminResourceScope::bucket_object(resource.warehouse, object_path.as_deref().unwrap_or("")), + ) + .await +} + +async fn table_catalog_request_principal(req: &S3Request) -> S3Result { + let Some(input_cred) = &req.credentials else { + return Err(s3_error!(InvalidRequest, "authentication required")); + }; + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + Ok(cred) +} + +async fn read_json_body(mut input: Body) -> S3Result { + let body = input + .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) + .await + .map_err(|err| s3_error!(InvalidRequest, "failed to read request body: {}", err))?; + if body.is_empty() { + return Err(s3_error!(InvalidRequest, "request body is required")); + } + serde_json::from_slice(&body).map_err(|err| s3_error!(InvalidRequest, "invalid JSON: {}", err)) +} + +async fn read_json_body_or_default(mut input: Body) -> S3Result +where + T: Default + DeserializeOwned, +{ + let body = input + .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) + .await + .map_err(|err| s3_error!(InvalidRequest, "failed to read request body: {}", err))?; + if body.is_empty() { + return Ok(T::default()); + } + serde_json::from_slice(&body).map_err(|err| s3_error!(InvalidRequest, "invalid JSON: {}", err)) +} + +fn warehouse_from_params(params: &Params<'_, '_>) -> S3Result { + let warehouse = params.get("warehouse").unwrap_or(""); + if warehouse.is_empty() { + return Err(s3_error!(InvalidRequest, "warehouse is required")); + } + Ok(warehouse.to_string()) +} + +fn warehouse_from_config_query(uri: &http::Uri) -> S3Result> { + let Some(query) = uri.query() else { + return Ok(None); + }; + let mut warehouse = None; + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + if key != WAREHOUSE_PROPERTY { + continue; + } + if warehouse.is_some() { + return Err(s3_error!(InvalidRequest, "warehouse query parameter must not be repeated")); + } + if value.is_empty() { + return Err(s3_error!(InvalidRequest, "warehouse query parameter must not be empty")); + } + warehouse = Some(value.into_owned()); + } + Ok(warehouse) +} + +fn rest_pagination_from_query(uri: &http::Uri, context: RestPageContext<'_>) -> S3Result { + let mut page_token = None; + let mut page_token_seen = false; + let mut page_size = None; + + if let Some(query) = uri.query() { + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + REST_PAGE_TOKEN_QUERY_PARAMETER => { + if page_token_seen { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "pageToken query parameter must not be repeated", + )); + } + page_token_seen = true; + page_token = Some(value.into_owned()); + } + REST_PAGE_SIZE_QUERY_PARAMETER => { + if page_size.is_some() { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "pageSize query parameter must not be repeated", + )); + } + let value = value.parse::().map_err(|_| { + iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "pageSize query parameter must be a positive integer", + ) + })?; + if value == 0 { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "pageSize query parameter must be greater than zero", + )); + } + page_size = Some(value.min(REST_MAX_PAGE_SIZE)); + } + _ => {} + } + } + } + + if !page_token_seen && page_size.is_none() { + return Ok(RestPagination::Unpaginated); + } + + let context = rest_page_context_fingerprint(context); + let cursor = match page_token.as_deref() { + None | Some("") => None, + Some(encoded) => Some(decode_rest_page_token(encoded, &context)?), + }; + let limit = NonZeroUsize::new(page_size.unwrap_or(REST_DEFAULT_PAGE_SIZE)).ok_or_else(|| { + iceberg_rest_error( + ICEBERG_ERROR_REST, + StatusCode::INTERNAL_SERVER_ERROR, + "REST page size must be greater than zero", + ) + })?; + Ok(RestPagination::Paginated { cursor, limit, context }) +} + +fn rest_page_context_fingerprint(context: RestPageContext<'_>) -> String { + let mut data = + Vec::with_capacity(context.resource.len() + context.warehouse.len() + context.namespace.map_or(0, str::len) + 2); + data.extend_from_slice(context.resource.as_bytes()); + data.push(0); + data.extend_from_slice(context.warehouse.as_bytes()); + data.push(0); + if let Some(namespace) = context.namespace { + data.extend_from_slice(namespace.as_bytes()); + } + hex_sha256(&data, str::to_string) +} + +fn decode_rest_page_token(encoded: &str, expected_context: &str) -> S3Result { + if encoded.len() > REST_PAGE_TOKEN_MAX_LENGTH { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "pageToken query parameter is too large", + )); + } + let data = base64_decode_url_safe_no_pad(encoded.as_bytes()).map_err(|_| { + iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "pageToken query parameter is malformed", + ) + })?; + let token = serde_json::from_slice::(&data).map_err(|_| { + iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "pageToken query parameter is malformed", + ) + })?; + if token.version != REST_PAGE_TOKEN_VERSION || token.context != expected_context || token.cursor.is_empty() { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "pageToken query parameter does not match this list operation", + )); + } + Ok(token.cursor) +} + +fn encode_rest_page_token(cursor: &str, context: &str) -> S3Result { + let token = RestPageToken { + version: REST_PAGE_TOKEN_VERSION, + context: context.to_string(), + cursor: cursor.to_string(), + }; + let data = serde_json::to_vec(&token).map_err(|err| { + iceberg_rest_error( + ICEBERG_ERROR_REST, + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to serialize REST page token: {err}"), + ) + })?; + let encoded = base64_encode_url_safe_no_pad(&data); + if encoded.len() > REST_PAGE_TOKEN_MAX_LENGTH { + return Err(iceberg_rest_error( + ICEBERG_ERROR_REST, + StatusCode::INTERNAL_SERVER_ERROR, + "REST page token exceeds the supported size", + )); + } + Ok(encoded) +} + +fn namespace_from_params(params: &Params<'_, '_>) -> S3Result { + let namespace = params.get("namespace").unwrap_or(""); + crate::table_catalog::Namespace::parse(namespace).map_err(|err| s3_error!(InvalidRequest, "invalid namespace: {}", err)) +} + +fn table_name_from_params(params: &Params<'_, '_>) -> S3Result { + let table = params.get("table").unwrap_or(""); + crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + Ok(table.to_string()) +} + +fn view_name_from_params(params: &Params<'_, '_>) -> S3Result { + let view = params.get("view").unwrap_or(""); + crate::table_catalog::IdentifierSegment::parse(view.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid view name: {}", err))?; + Ok(view.to_string()) +} + +fn ref_name_from_params(params: &Params<'_, '_>) -> S3Result { + let ref_name = params.get("ref").unwrap_or(""); + crate::table_catalog::IdentifierSegment::parse(ref_name.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid ref name: {}", err))?; + Ok(ref_name.to_string()) +} + +fn job_id_from_params(params: &Params<'_, '_>) -> S3Result { + let job = params.get("job").unwrap_or(""); + if job.is_empty() { + return Err(s3_error!(InvalidRequest, "maintenance job id is required")); + } + Ok(job.to_string()) +} + +fn table_catalog_backend() -> S3Result> { + let store = current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store not initialized"))?; + Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new(store)) +} + +type EcStoreObjectTableCatalogStore = + crate::table_catalog::ObjectTableCatalogStore>; + +fn table_catalog_store_from_backend( + backend: crate::table_catalog::EcStoreTableCatalogObjectBackend, +) -> S3Result> { + crate::table_catalog::ConfiguredTableCatalogStore::from_env(backend).map_err(catalog_store_error) +} + +fn table_catalog_store() -> S3Result> { + let backend = table_catalog_backend()?; + table_catalog_store_from_backend(backend) +} + +fn table_catalog_object_store() -> S3Result { + match crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)? { + crate::table_catalog::TableCatalogBackingMode::ObjectBacked => { + let backend = table_catalog_backend()?; + Ok(crate::table_catalog::ObjectTableCatalogStore::new(backend)) + } + crate::table_catalog::TableCatalogBackingMode::DurableStrong => Err(s3_error!( + InvalidRequest, + "operation is not supported with {} table catalog backing", + crate::table_catalog::TABLE_CATALOG_BACKING_DURABLE_STRONG + )), + } +} + +async fn table_bucket_enabled_from_metadata(bucket: &str) -> S3Result { + let metadata = metadata_sys::get(bucket) + .await + .map_err(|err| s3_error!(InvalidRequest, "failed to load table bucket metadata for {bucket}: {}", err))?; + Ok(metadata.table_bucket_enabled()) +} + +async fn ensure_table_bucket_enabled(bucket: &str) -> S3Result<()> { + if table_bucket_enabled_from_metadata(bucket).await? { + return Ok(()); + } + Err(s3_error!(InvalidRequest, "bucket {bucket} is not table-enabled")) +} + +fn table_bucket_entry_from_metadata_marker(bucket: &str) -> crate::table_catalog::TableBucketEntry { + crate::table_catalog::TableBucketEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + catalog_type: crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(), + warehouse_root: format!("s3://{bucket}/"), + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + } +} + +async fn enable_table_bucket_marker(bucket: &str) -> S3Result<()> { + let marker = crate::table_catalog::table_bucket_marker_json() + .map_err(|err| s3_error!(InternalError, "failed to serialize table bucket marker: {}", err))?; + metadata_sys::update(bucket, crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG, marker) + .await + .map(|_| ()) + .map_err(|err| s3_error!(InvalidRequest, "failed to enable table bucket {bucket}: {}", err)) +} + +async fn ensure_table_bucket_entry(store: &S, bucket: &str, table_bucket_enabled: bool) -> S3Result<()> +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + if !table_bucket_enabled { + return Err(s3_error!(InvalidRequest, "bucket {bucket} is not table-enabled")); + } + if store.get_table_bucket(bucket).await.map_err(catalog_store_error)?.is_some() { + return Ok(()); + } + store + .put_table_bucket(table_bucket_entry_from_metadata_marker(bucket)) + .await + .map_err(catalog_store_error) +} + +async fn table_bucket_response(store: &S, bucket: &str, enabled: bool) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let entry = store.get_table_bucket(bucket).await.map_err(catalog_store_error)?; + let (catalog_type, warehouse_location, properties, catalog_entry_present) = match entry { + Some(entry) => (entry.catalog_type, entry.warehouse_root, entry.properties, true), + None => ( + crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(), + format!("s3://{bucket}/"), + BTreeMap::new(), + false, + ), + }; + + Ok(TableBucketResponse { + table_bucket: bucket.to_string(), + enabled, + catalog_type, + warehouse: bucket.to_string(), + warehouse_location: warehouse_location.clone(), + catalog_uri: format!("{TABLE_CATALOG_PREFIX}/{bucket}"), + compat_catalog_uri: format!("{TABLE_CATALOG_COMPAT_PREFIX}/{bucket}"), + credential_vending: CREDENTIAL_VENDING_UNSUPPORTED, + credential_scope: CREDENTIAL_SCOPE_WAREHOUSE_PREFIX, + credential_scope_prefix: warehouse_location, + catalog_entry_present, + properties, + }) +} + +async fn enable_table_bucket_response(store: &S, bucket: &str) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + enable_table_bucket_marker(bucket).await?; + ensure_table_bucket_entry(store, bucket, true).await?; + table_bucket_response(store, bucket, true).await +} + +fn namespace_segments(namespace: &crate::table_catalog::Namespace) -> Vec { + namespace + .segments() + .iter() + .map(|segment| segment.as_str().to_string()) + .collect() +} + +fn namespace_from_segments(segments: &[String]) -> S3Result { + if segments.is_empty() { + return Err(s3_error!(InvalidRequest, "namespace cannot be empty")); + } + + let namespace = segments.join("."); + crate::table_catalog::Namespace::parse(&namespace).map_err(|err| s3_error!(InvalidRequest, "invalid namespace: {}", err)) +} + +fn namespace_response_from_entry(entry: crate::table_catalog::NamespaceEntry) -> S3Result { + let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) + .map_err(|err| s3_error!(InternalError, "persisted namespace entry is invalid: {}", err))?; + Ok(RestNamespaceResponse { + namespace: namespace_segments(&namespace), + properties: entry.properties, + }) +} + +fn list_namespaces_response_from_entries( + entries: Vec, + next_page_token: Option, +) -> S3Result { + let namespaces = entries + .into_iter() + .map(|entry| { + let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) + .map_err(|err| s3_error!(InternalError, "persisted namespace entry is invalid: {}", err))?; + Ok(namespace_segments(&namespace)) + }) + .collect::>>()?; + Ok(RestListNamespacesResponse { + namespaces, + next_page_token, + }) +} + +fn list_tables_response_from_entries( + entries: Vec, + next_page_token: Option, +) -> S3Result { + let identifiers = entries + .into_iter() + .map(|entry| { + let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) + .map_err(|err| s3_error!(InternalError, "persisted table entry namespace is invalid: {}", err))?; + Ok(RestTableIdentifier { + namespace: namespace_segments(&namespace), + name: entry.table, + }) + }) + .collect::>>()?; + Ok(RestListTablesResponse { + identifiers, + next_page_token, + }) +} + +fn list_views_response_from_entries( + entries: Vec, + next_page_token: Option, +) -> S3Result { + let identifiers = entries + .into_iter() + .map(|entry| { + let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) + .map_err(|err| s3_error!(InternalError, "persisted view entry namespace is invalid: {}", err))?; + Ok(RestTableIdentifier { + namespace: namespace_segments(&namespace), + name: entry.view, + }) + }) + .collect::>>()?; + Ok(RestListViewsResponse { + identifiers, + next_page_token, + }) +} + +fn table_credential_vending_enabled() -> bool { + std::env::var(ENV_TABLE_CATALOG_CREDENTIAL_VENDING) + .ok() + .map(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true" | "on" | "enabled")) + .unwrap_or(false) +} + +fn table_credential_ttl_seconds() -> i64 { + std::env::var(ENV_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS) + .ok() + .and_then(|value| value.parse::().ok()) + .map(|seconds| seconds.clamp(MIN_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS, MAX_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS)) + .unwrap_or(DEFAULT_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS) +} + +fn table_credential_scope(entry: &crate::table_catalog::TableEntry) -> S3Result { + let location = entry + .warehouse_location + .strip_prefix("s3://") + .ok_or_else(|| s3_error!(InvalidRequest, "table warehouse location must be an s3 URI"))?; + let (bucket, object_prefix) = location + .split_once('/') + .ok_or_else(|| s3_error!(InvalidRequest, "table warehouse location must include an object prefix"))?; + if bucket != entry.table_bucket { + return Err(s3_error!(InvalidRequest, "table warehouse location must be inside the table bucket")); + } + let object_prefix = normalize_table_credential_object_prefix(object_prefix)?; + Ok(TableCredentialScope { + scope_prefix: format!("s3://{bucket}/{object_prefix}"), + object_prefix, + }) +} + +fn normalize_table_credential_object_prefix(object_prefix: &str) -> S3Result { + let object_prefix = object_prefix.strip_suffix('/').unwrap_or(object_prefix); + if object_prefix.is_empty() { + return Err(s3_error!(InvalidRequest, "table credential scope prefix is empty")); + } + if object_prefix.contains('\\') { + return Err(s3_error!( + InvalidRequest, + "table credential scope prefix contains an invalid path separator" + )); + } + if object_prefix + .split('/') + .any(|segment| segment.is_empty() || segment == "." || segment == "..") + { + return Err(s3_error!( + InvalidRequest, + "table credential scope prefix contains an invalid path segment" + )); + } + + let mut normalized = object_prefix.to_string(); + normalized.push('/'); + Ok(normalized) +} + +fn table_credential_catalog_resource(entry: &crate::table_catalog::TableEntry) -> S3Result { + let namespace = crate::table_catalog::Namespace::parse(&entry.namespace) + .map_err(|err| s3_error!(InvalidRequest, "invalid table credential namespace: {}", err))?; + let table = crate::table_catalog::IdentifierSegment::parse(&entry.table) + .map_err(|err| s3_error!(InvalidRequest, "invalid table credential table name: {}", err))?; + Ok(format!("namespaces/{}/tables/{}", namespace.storage_id(), table.as_str())) +} + +fn table_credential_session_policy(entry: &crate::table_catalog::TableEntry, object_prefix: &str) -> S3Result { + let bucket = &entry.table_bucket; + let object_prefix = normalize_table_credential_object_prefix(object_prefix)?; + let catalog_resource = table_credential_catalog_resource(entry)?; + let policy = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + format!("arn:aws:s3:::{bucket}/{object_prefix}*") + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation" + ], + "Resource": [ + format!("arn:aws:s3:::{bucket}") + ] + }, + { + "Effect": "Allow", + "Action": [ + "admin:GetTableMetadata", + "admin:SetTableMetadata" + ], + "Resource": [ + format!("arn:aws:s3:::{bucket}/{catalog_resource}") + ] + } + ] + }); + let data = serde_json::to_vec(&policy) + .map_err(|err| s3_error!(InternalError, "failed to serialize table credential policy: {}", err))?; + Policy::parse_config(&data).map_err(|err| s3_error!(InvalidRequest, "invalid table credential policy: {}", err)) +} + +fn storage_credential_from_issued(scope: TableCredentialScope, issued: IssuedTableCredentials) -> RestStorageCredential { + let mut config = BTreeMap::new(); + config.insert(S3_ACCESS_KEY_ID_CONFIG_KEY.to_string(), issued.access_key_id); + config.insert(S3_SECRET_ACCESS_KEY_CONFIG_KEY.to_string(), issued.secret_access_key); + config.insert(S3_SESSION_TOKEN_CONFIG_KEY.to_string(), issued.session_token); + config.insert(CREDENTIAL_VENDING_CONFIG_KEY.to_string(), CREDENTIAL_VENDING_SUPPORTED.to_string()); + config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CATALOG_VENDED.to_string()); + config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); + config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), scope.scope_prefix.clone()); + config.insert( + CREDENTIAL_EXPIRATION_CONFIG_KEY.to_string(), + issued.expiration.unix_timestamp().to_string(), + ); + RestStorageCredential { + prefix: scope.scope_prefix, + config, + } +} + +fn table_metadata_location_for_client(table_bucket: &str, metadata_location: &str) -> String { + if crate::table_catalog::is_reserved_table_object_key(metadata_location) { + format!("s3://{table_bucket}/{metadata_location}") + } else { + metadata_location.to_string() + } +} + +fn table_metadata_location_for_catalog(table_bucket: &str, metadata_location: &str) -> S3Result { + crate::table_catalog::table_catalog_object_key_from_location(table_bucket, metadata_location) + .ok_or_else(|| s3_error!(InvalidRequest, "metadata location must be inside the table bucket")) +} + +fn table_metadata_for_client(table_bucket: &str, mut metadata: serde_json::Value) -> serde_json::Value { + if let Some(metadata_log) = metadata.get_mut("metadata-log").and_then(serde_json::Value::as_array_mut) { + for entry in metadata_log { + let Some(metadata_file) = entry.get_mut("metadata-file") else { + continue; + }; + let Some(metadata_location) = metadata_file.as_str() else { + continue; + }; + if crate::table_catalog::is_reserved_table_object_key(metadata_location) { + *metadata_file = serde_json::Value::String(table_metadata_location_for_client(table_bucket, metadata_location)); + } + } + } + + metadata +} + +fn load_table_response_from_entry(entry: crate::table_catalog::TableEntry, metadata: serde_json::Value) -> RestLoadTableResponse { + let mut config = BTreeMap::new(); + let warehouse_location = entry.warehouse_location.clone(); + let metadata_location = table_metadata_location_for_client(&entry.table_bucket, &entry.metadata_location); + let metadata = table_metadata_for_client(&entry.table_bucket, metadata); + config.insert("warehouse-location".to_string(), warehouse_location.clone()); + config.insert(CREDENTIAL_VENDING_CONFIG_KEY.to_string(), CREDENTIAL_VENDING_UNSUPPORTED.to_string()); + config.insert( + CREDENTIAL_VENDING_REASON_CONFIG_KEY.to_string(), + CREDENTIAL_VENDING_UNSUPPORTED_REASON.to_string(), + ); + config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); + config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), warehouse_location); + config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()); + + RestLoadTableResponse { + metadata_location, + metadata, + config, + storage_credentials: Vec::new(), + } +} + +fn load_view_response_from_entry(entry: crate::table_catalog::ViewEntry, metadata: serde_json::Value) -> RestLoadViewResponse { + let mut config = BTreeMap::new(); + let warehouse_location = entry.warehouse_location.clone(); + config.insert("warehouse-location".to_string(), warehouse_location.clone()); + config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); + config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), warehouse_location); + config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()); + + RestLoadViewResponse { + metadata_location: entry.metadata_location, + metadata, + config, + } +} + +fn load_credentials_response_config(vending: &str, mode: &str, reason: Option<&str>) -> BTreeMap { + let mut config = BTreeMap::new(); + config.insert(CREDENTIAL_VENDING_CONFIG_KEY.to_string(), vending.to_string()); + config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), mode.to_string()); + if let Some(reason) = reason { + config.insert(CREDENTIAL_VENDING_REASON_CONFIG_KEY.to_string(), reason.to_string()); + } + config +} + +fn add_table_credential_scope_config(config: &mut BTreeMap, scope_prefix: &str) { + config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); + config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), scope_prefix.to_string()); +} + +async fn load_credentials_response_from_entry( + entry: &crate::table_catalog::TableEntry, + issuer: &dyn TableCredentialIssuer, + principal: Option<&rustfs_credentials::Credentials>, +) -> S3Result { + if !issuer.enabled() { + return Ok(RestLoadCredentialsResponse { + config: load_credentials_response_config( + CREDENTIAL_VENDING_UNSUPPORTED, + CREDENTIAL_MODE_CLIENT_PROVIDED, + Some(CREDENTIAL_VENDING_DISABLED_REASON), + ), + storage_credentials: Vec::new(), + }); + } + let scope = table_credential_scope(entry)?; + let request = TableCredentialIssueRequest { + entry, + principal, + scope_prefix: scope.scope_prefix.clone(), + object_prefix: scope.object_prefix.clone(), + }; + let scope_prefix = scope.scope_prefix.clone(); + let storage_credentials = match issuer.issue_table_credentials(request).await? { + Some(issued) => vec![storage_credential_from_issued(scope, issued)], + None => { + let mut config = load_credentials_response_config( + CREDENTIAL_VENDING_UNSUPPORTED, + CREDENTIAL_MODE_CLIENT_PROVIDED, + Some(CREDENTIAL_VENDING_UNSUPPORTED_REASON), + ); + add_table_credential_scope_config(&mut config, &scope_prefix); + return Ok(RestLoadCredentialsResponse { + config, + storage_credentials: Vec::new(), + }); + } + }; + let mut config = load_credentials_response_config(CREDENTIAL_VENDING_SUPPORTED, CREDENTIAL_MODE_CATALOG_VENDED, None); + add_table_credential_scope_config(&mut config, &scope_prefix); + Ok(RestLoadCredentialsResponse { + config, + storage_credentials, + }) +} + +fn commit_table_response_from_result( + result: crate::table_catalog::TableCommitResult, + metadata: serde_json::Value, +) -> RestCommitTableResponse { + let metadata_location = table_metadata_location_for_client(&result.table.table_bucket, &result.table.metadata_location); + let metadata = table_metadata_for_client(&result.table.table_bucket, metadata); + RestCommitTableResponse { + metadata_location, + metadata, + version_token: result.table.version_token, + generation: result.table.generation, + commit_id: result.commit_log.commit_id, + } +} + +fn table_metadata_location_response_from_entry(entry: crate::table_catalog::TableEntry) -> TableMetadataLocationResponse { + let metadata_location = table_metadata_location_for_client(&entry.table_bucket, &entry.metadata_location); + TableMetadataLocationResponse { + metadata_location, + version_token: entry.version_token, + generation: entry.generation, + warehouse_location: entry.warehouse_location, + } +} + +fn table_commit_request_from_rest_request( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: RestCommitTableRequest, +) -> S3Result { + let expected_metadata_location = request + .expected_metadata_location + .ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires expected-metadata-location"))?; + let new_metadata_location = request + .new_metadata_location + .ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires new-metadata-location"))?; + Ok(crate::table_catalog::TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + commit_id: request.commit_id.unwrap_or_else(|| Uuid::new_v4().to_string()), + idempotency_key: request.idempotency_key, + operation: request.operation.unwrap_or_else(|| "commit".to_string()), + expected_version_token: request + .expected_version_token + .ok_or_else(|| s3_error!(InvalidRequest, "legacy commit requires expected-version-token"))?, + expected_metadata_location: table_metadata_location_for_catalog(bucket, &expected_metadata_location)?, + new_metadata_location: table_metadata_location_for_catalog(bucket, &new_metadata_location)?, + requirements: request.requirements, + writer: request.writer, + }) +} + +fn validate_table_location_in_bucket(bucket: &str, location: &str) -> S3Result<()> { + crate::table_catalog::validate_table_warehouse_location(bucket, location).map_err(catalog_store_error) +} + +fn validate_view_location_in_bucket(bucket: &str, location: &str) -> S3Result<()> { + crate::table_catalog::validate_view_warehouse_location(bucket, location).map_err(catalog_store_error) +} + +fn metadata_table_uuid(metadata: &serde_json::Value) -> S3Result<&str> { + metadata + .get("table-uuid") + .and_then(serde_json::Value::as_str) + .filter(|uuid| !uuid.is_empty()) + .ok_or_else(|| s3_error!(InvalidRequest, "table metadata is missing table-uuid")) +} + +fn metadata_format_version(metadata: &serde_json::Value) -> S3Result { + let version = metadata + .get("format-version") + .and_then(serde_json::Value::as_u64) + .filter(|version| *version > 0) + .ok_or_else(|| s3_error!(InvalidRequest, "table metadata is missing format-version"))?; + u16::try_from(version).map_err(|_| s3_error!(InvalidRequest, "table metadata format-version is too large")) +} + +fn metadata_table_location(metadata: &serde_json::Value) -> S3Result<&str> { + metadata + .get("location") + .and_then(serde_json::Value::as_str) + .filter(|location| !location.is_empty()) + .ok_or_else(|| s3_error!(InvalidRequest, "table metadata is missing location")) +} + +fn validate_metadata_table_location_in_bucket(bucket: &str, metadata: &serde_json::Value) -> S3Result<()> { + let location = metadata_table_location(metadata)?; + validate_table_location_in_bucket(bucket, location) +} + +fn validate_metadata_view_location_in_bucket(bucket: &str, metadata: &serde_json::Value) -> S3Result<()> { + let location = metadata_table_location(metadata)?; + validate_view_location_in_bucket(bucket, location) +} + +fn validate_metadata_matches_current_metadata( + current_metadata: &serde_json::Value, + target_metadata: &serde_json::Value, +) -> S3Result<()> { + let expected_table_uuid = metadata_table_uuid(current_metadata)?; + metadata_format_version(current_metadata)?; + let target_table_uuid = metadata_table_uuid(target_metadata)?; + metadata_format_version(target_metadata)?; + if target_table_uuid != expected_table_uuid { + return Err(s3_error!( + InvalidRequest, + "table metadata table-uuid does not match current table metadata" + )); + } + Ok(()) +} + +fn metadata_view_uuid(metadata: &serde_json::Value) -> S3Result<&str> { + metadata + .get("view-uuid") + .and_then(serde_json::Value::as_str) + .filter(|uuid| !uuid.is_empty()) + .ok_or_else(|| s3_error!(InvalidRequest, "view metadata is missing view-uuid")) +} + +fn validate_metadata_matches_current_view_metadata( + current_metadata: &serde_json::Value, + target_metadata: &serde_json::Value, +) -> S3Result<()> { + let expected_view_uuid = metadata_view_uuid(current_metadata)?; + metadata_format_version(current_metadata)?; + let target_view_uuid = metadata_view_uuid(target_metadata)?; + metadata_format_version(target_metadata)?; + if target_view_uuid != expected_view_uuid { + return Err(s3_error!(InvalidRequest, "view metadata view-uuid does not match current view metadata")); + } + Ok(()) +} + +fn adopt_registered_metadata_identity( + entry: &mut crate::table_catalog::TableEntry, + metadata: &serde_json::Value, +) -> S3Result<()> { + entry.table_uuid = metadata_table_uuid(metadata)?.to_string(); + entry.format_version = metadata_format_version(metadata)?; + entry.warehouse_location = metadata_table_location(metadata)?.to_string(); + Ok(()) +} + +fn table_entry_from_register_request( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + request: RegisterTableRequest, +) -> S3Result { + if request.overwrite { + return Err(s3_error!(NotImplemented, "register table overwrite is not supported")); + } + let table = crate::table_catalog::IdentifierSegment::parse(request.name) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; + if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, &metadata_location) { + return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); + } + + let table_id = Uuid::new_v4().to_string(); + Ok(crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: table_id.clone(), + table_uuid: Uuid::new_v4().to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: format!("s3://{bucket}/tables/{table_id}"), + metadata_location, + version_token: format!("token-{}", Uuid::new_v4()), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) +} + +fn table_entry_from_import_request( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: CatalogImportRequest, +) -> S3Result { + let table = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; + if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, &metadata_location) { + return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); + } + + let table_id = Uuid::new_v4().to_string(); + Ok(crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: table_id.clone(), + table_uuid: Uuid::new_v4().to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: format!("s3://{bucket}/tables/{table_id}"), + metadata_location, + version_token: format!("token-{}", Uuid::new_v4()), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: request.properties, + created_at: None, + updated_at: None, + }) +} + +fn table_entry_from_create_table_request( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + request: CreateTableRequest, +) -> S3Result<(crate::table_catalog::TableEntry, serde_json::Value)> { + if request.stage_create { + return Err(s3_error!(NotImplemented, "stage-create is not supported")); + } + + let table = crate::table_catalog::IdentifierSegment::parse(request.name) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + let table_id = Uuid::new_v4().to_string(); + let table_uuid = Uuid::new_v4().to_string(); + let warehouse_location = request.location.unwrap_or_else(|| format!("s3://{bucket}/tables/{table_id}")); + validate_table_location_in_bucket(bucket, &warehouse_location)?; + let metadata_location = + crate::table_catalog::default_table_metadata_file_path(namespace, &table, &next_metadata_file_name(1, &table_id)); + + let mut entry = crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id, + table_uuid, + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location, + metadata_location, + version_token: format!("token-{}", Uuid::new_v4()), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: request.properties, + created_at: None, + updated_at: None, + }; + let metadata = initial_table_metadata_json( + &entry, + request.schema, + request.partition_spec, + request.write_order, + entry.properties.clone(), + )?; + entry.format_version = metadata + .get("format-version") + .and_then(serde_json::Value::as_u64) + .and_then(|version| u16::try_from(version).ok()) + .unwrap_or(2); + Ok((entry, metadata)) +} + +fn view_entry_from_create_view_request( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + request: CreateViewRequest, +) -> S3Result<(crate::table_catalog::ViewEntry, serde_json::Value)> { + let view = crate::table_catalog::IdentifierSegment::parse(request.name) + .map_err(|err| s3_error!(InvalidRequest, "invalid view name: {}", err))?; + let view_id = Uuid::new_v4().to_string(); + let view_uuid = Uuid::new_v4().to_string(); + let warehouse_location = request.location.unwrap_or_else(|| format!("s3://{bucket}/views/{view_id}")); + validate_view_location_in_bucket(bucket, &warehouse_location)?; + let metadata_location = + crate::table_catalog::default_view_metadata_file_path(namespace, &view, &next_metadata_file_name(1, &view_id)); + + let entry = crate::table_catalog::ViewEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + view_id, + view_uuid, + format: "ICEBERG_VIEW".to_string(), + format_version: 1, + warehouse_location, + metadata_location, + version_token: format!("token-{}", Uuid::new_v4()), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: request.properties, + created_at: None, + updated_at: None, + }; + let metadata = initial_view_metadata_json(&entry, request.schema, request.view_version, entry.properties.clone())?; + Ok((entry, metadata)) +} + +fn initial_table_metadata_json( + entry: &crate::table_catalog::TableEntry, + mut schema: serde_json::Value, + partition_spec: Option, + write_order: Option, + properties: BTreeMap, +) -> S3Result { + let schema_object = schema + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "schema must be a JSON object"))?; + schema_object + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(0)); + let schema_id = schema_object + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "schema-id must be an integer"))?; + let last_column_id = max_field_id(&schema); + + let mut spec = partition_spec.unwrap_or_else(|| { + serde_json::json!({ + "spec-id": 0, + "fields": [] + }) + }); + let spec_object = spec + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "partition-spec must be a JSON object"))?; + spec_object + .entry("spec-id".to_string()) + .or_insert_with(|| serde_json::Value::from(0)); + spec_object + .entry("fields".to_string()) + .or_insert_with(|| serde_json::Value::Array(Vec::new())); + let spec_id = spec_object + .get("spec-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition spec-id must be an integer"))?; + let last_partition_id = max_partition_field_id(&spec); + + let mut sort_order = write_order.unwrap_or_else(|| { + serde_json::json!({ + "order-id": 0, + "fields": [] + }) + }); + let sort_order_object = sort_order + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "write-order must be a JSON object"))?; + sort_order_object + .entry("order-id".to_string()) + .or_insert_with(|| serde_json::Value::from(0)); + sort_order_object + .entry("fields".to_string()) + .or_insert_with(|| serde_json::Value::Array(Vec::new())); + let sort_order_id = sort_order_object + .get("order-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "sort order-id must be an integer"))?; + + Ok(serde_json::json!({ + "format-version": entry.format_version, + "table-uuid": entry.table_uuid, + "location": entry.warehouse_location, + "last-sequence-number": 0, + "last-updated-ms": current_time_millis(), + "last-column-id": last_column_id, + "schemas": [schema], + "current-schema-id": schema_id, + "partition-specs": [spec], + "default-spec-id": spec_id, + "last-partition-id": last_partition_id, + "sort-orders": [sort_order], + "default-sort-order-id": sort_order_id, + "properties": properties, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {} + })) +} + +fn initial_view_metadata_json( + entry: &crate::table_catalog::ViewEntry, + mut schema: serde_json::Value, + mut view_version: serde_json::Value, + properties: BTreeMap, +) -> S3Result { + let schema_object = schema + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "schema must be a JSON object"))?; + schema_object + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(0)); + let schema_id = schema_object + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "schema-id must be an integer"))?; + + let view_version_object = view_version + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))?; + view_version_object + .entry("version-id".to_string()) + .or_insert_with(|| serde_json::Value::from(1)); + view_version_object + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(schema_id)); + view_version_object + .entry("timestamp-ms".to_string()) + .or_insert_with(|| serde_json::Value::from(current_time_millis())); + let version_id = view_version_object + .get("version-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "view-version version-id must be an integer"))?; + let timestamp_ms = view_version_object + .get("timestamp-ms") + .and_then(serde_json::Value::as_i64) + .unwrap_or_else(current_time_millis); + + Ok(serde_json::json!({ + "format-version": entry.format_version, + "view-uuid": entry.view_uuid, + "location": entry.warehouse_location, + "current-version-id": version_id, + "schemas": [schema], + "versions": [view_version], + "version-log": [{ + "timestamp-ms": timestamp_ms, + "version-id": version_id + }], + "metadata-log": [], + "properties": properties + })) +} + +fn current_time_millis() -> i64 { + let now = OffsetDateTime::now_utc(); + now.unix_timestamp() + .saturating_mul(1000) + .saturating_add(i64::from(now.millisecond())) +} + +fn max_field_id(value: &serde_json::Value) -> i64 { + let mut max_id = 0; + collect_max_field_id(value, &mut max_id); + max_id +} + +fn collect_max_field_id(value: &serde_json::Value, max_id: &mut i64) { + match value { + serde_json::Value::Object(object) => { + if let Some(id) = object.get("id").and_then(serde_json::Value::as_i64) { + *max_id = (*max_id).max(id); + } + for child in object.values() { + collect_max_field_id(child, max_id); + } + } + serde_json::Value::Array(values) => { + for child in values { + collect_max_field_id(child, max_id); + } + } + _ => {} + } +} + +fn max_partition_field_id(value: &serde_json::Value) -> i64 { + let mut max_id = 999; + let Some(fields) = value.get("fields").and_then(serde_json::Value::as_array) else { + return max_id; + }; + for field in fields { + if let Some(field_id) = field.get("field-id").and_then(serde_json::Value::as_i64) { + max_id = max_id.max(field_id); + } + } + max_id +} + +fn standard_commit_ids(commit_id: Option) -> (String, String) { + match commit_id { + Some(commit_id) => match Uuid::parse_str(&commit_id) { + Ok(uuid) => { + let commit_id = uuid.to_string(); + (commit_id.clone(), commit_id) + } + Err(_) => { + let metadata_file_token = table_catalog_path_hash(&commit_id); + (commit_id, metadata_file_token) + } + }, + None => { + let commit_id = Uuid::new_v4().to_string(); + (commit_id.clone(), commit_id) + } + } +} + +fn next_metadata_file_name(generation: u64, metadata_file_token: &str) -> String { + format!("{generation:05}-{metadata_file_token}.metadata.json") +} + +fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements: &[serde_json::Value]) -> S3Result<()> { + for requirement in requirements { + let requirement_type = requirement + .get("type") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "commit requirement type is required"))?; + match requirement_type { + "assert-create" => { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: table already exists")); + } + "assert-table-uuid" => { + let expected = requirement + .get("uuid") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "assert-table-uuid requires uuid"))?; + let actual = metadata + .get("table-uuid") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing table-uuid"))?; + if actual != expected { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: table uuid changed")); + } + } + "assert-current-schema-id" => { + validate_i64_requirement(metadata, requirement, "current-schema-id", "current schema id")?; + } + "assert-default-spec-id" => { + validate_i64_requirement(metadata, requirement, "default-spec-id", "default spec id")?; + } + "assert-default-sort-order-id" => { + validate_i64_requirement(metadata, requirement, "default-sort-order-id", "default sort order id")?; + } + "assert-last-assigned-field-id" => { + validate_i64_requirement_with_metadata_key( + metadata, + requirement, + "last-assigned-field-id", + "last-column-id", + "last assigned field id", + )?; + } + "assert-last-assigned-partition-id" => { + validate_i64_requirement_with_metadata_key( + metadata, + requirement, + "last-assigned-partition-id", + "last-partition-id", + "last assigned partition id", + )?; + } + "assert-ref-snapshot-id" => validate_ref_snapshot_requirement(metadata, requirement)?, + "assert-current-snapshot-id" => validate_current_snapshot_requirement(metadata, requirement)?, + _ => return Err(s3_error!(NotImplemented, "unsupported commit requirement: {requirement_type}")), + } + } + Ok(()) +} + +fn validate_i64_requirement( + metadata: &serde_json::Value, + requirement: &serde_json::Value, + key: &str, + label: &str, +) -> S3Result<()> { + validate_i64_requirement_with_metadata_key(metadata, requirement, key, key, label) +} + +fn validate_i64_requirement_with_metadata_key( + metadata: &serde_json::Value, + requirement: &serde_json::Value, + requirement_key: &str, + metadata_key: &str, + label: &str, +) -> S3Result<()> { + let expected = requirement + .get(requirement_key) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "{requirement_key} must be an integer"))?; + let actual = metadata + .get(metadata_key) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing {metadata_key}"))?; + if actual != expected { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: {label} changed")); + } + Ok(()) +} + +fn validate_ref_snapshot_requirement(metadata: &serde_json::Value, requirement: &serde_json::Value) -> S3Result<()> { + let ref_name = requirement + .get("ref") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "assert-ref-snapshot-id requires ref"))?; + let refs = metadata.get("refs").and_then(serde_json::Value::as_object); + let actual = refs + .and_then(|refs| refs.get(ref_name)) + .and_then(|reference| reference.get("snapshot-id")) + .and_then(serde_json::Value::as_i64); + if requirement.get("snapshot-id").is_some_and(serde_json::Value::is_null) { + if actual.is_some() { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: snapshot ref exists")); + } + return Ok(()); + } + let expected = requirement + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "assert-ref-snapshot-id requires snapshot-id"))?; + if actual != Some(expected) { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: snapshot ref changed")); + } + Ok(()) +} + +fn validate_current_snapshot_requirement(metadata: &serde_json::Value, requirement: &serde_json::Value) -> S3Result<()> { + let actual = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); + if requirement.get("snapshot-id").is_some_and(serde_json::Value::is_null) { + if actual.is_some() { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: current snapshot exists")); + } + return Ok(()); + } + let expected = requirement + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "assert-current-snapshot-id requires snapshot-id"))?; + if actual != Some(expected) { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: current snapshot changed")); + } + Ok(()) +} + +fn apply_table_commit_updates( + mut metadata: serde_json::Value, + updates: &[serde_json::Value], + previous_metadata_location: &str, +) -> S3Result { + if !metadata.is_object() { + return Err(s3_error!(InvalidRequest, "current table metadata must be a JSON object")); + } + + for update in updates { + let action = update + .get("action") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "table update action is required"))?; + match action { + "assign-uuid" => apply_assign_uuid_update(&mut metadata, update)?, + "upgrade-format-version" => apply_upgrade_format_version_update(&mut metadata, update)?, + "add-schema" => apply_add_schema_update(&mut metadata, update)?, + "set-current-schema" => apply_set_current_schema_update(&mut metadata, update)?, + "add-spec" => apply_add_spec_update(&mut metadata, update)?, + "set-default-spec" => apply_set_default_spec_update(&mut metadata, update)?, + "add-sort-order" => apply_add_sort_order_update(&mut metadata, update)?, + "set-default-sort-order" => apply_set_default_sort_order_update(&mut metadata, update)?, + "add-snapshot" => apply_add_snapshot_update(&mut metadata, update)?, + "set-snapshot-ref" => apply_set_snapshot_ref_update(&mut metadata, update)?, + "remove-snapshots" => apply_remove_snapshots_update(&mut metadata, update)?, + "remove-snapshot-ref" => apply_remove_snapshot_ref_update(&mut metadata, update)?, + "set-location" => apply_set_location_update(&mut metadata, update)?, + "set-properties" => apply_set_properties_update(&mut metadata, update)?, + "remove-properties" => apply_remove_properties_update(&mut metadata, update)?, + _ => return Err(s3_error!(NotImplemented, "unsupported table update: {action}")), + } + } + + append_previous_metadata_log(&mut metadata, previous_metadata_location)?; + metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(current_time_millis())); + Ok(metadata) +} + +fn validate_view_commit_requirements(metadata: &serde_json::Value, requirements: &[serde_json::Value]) -> S3Result<()> { + for requirement in requirements { + let requirement_type = requirement + .get("type") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "commit requirement type is required"))?; + match requirement_type { + "assert-view-uuid" => { + let expected = requirement + .get("uuid") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "assert-view-uuid requires uuid"))?; + let actual = metadata + .get("view-uuid") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "current view metadata is missing view-uuid"))?; + if actual != expected { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: view uuid changed")); + } + } + "assert-current-view-version-id" => { + validate_i64_requirement_with_metadata_key( + metadata, + requirement, + "current-view-version-id", + "current-version-id", + "current view version id", + )?; + } + _ => return Err(s3_error!(NotImplemented, "unsupported view commit requirement: {requirement_type}")), + } + } + Ok(()) +} + +fn apply_view_commit_updates( + mut metadata: serde_json::Value, + updates: &[serde_json::Value], + previous_metadata_location: &str, +) -> S3Result { + if !metadata.is_object() { + return Err(s3_error!(InvalidRequest, "current view metadata must be a JSON object")); + } + + for update in updates { + let action = update + .get("action") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "view update action is required"))?; + match action { + "assign-uuid" => apply_assign_uuid_update(&mut metadata, update)?, + "add-schema" => apply_add_schema_update(&mut metadata, update)?, + "set-current-schema" => apply_set_current_schema_update(&mut metadata, update)?, + "add-view-version" => apply_add_view_version_update(&mut metadata, update)?, + "set-current-view-version" => apply_set_current_view_version_update(&mut metadata, update)?, + "set-location" => apply_set_location_update(&mut metadata, update)?, + "set-properties" => apply_set_properties_update(&mut metadata, update)?, + "remove-properties" => apply_remove_properties_update(&mut metadata, update)?, + _ => return Err(s3_error!(NotImplemented, "unsupported view update: {action}")), + } + } + + append_previous_metadata_log(&mut metadata, previous_metadata_location)?; + metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(current_time_millis())); + Ok(metadata) +} + +fn apply_assign_uuid_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let uuid = update + .get("uuid") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "assign-uuid requires uuid"))?; + let object = metadata_object_mut(metadata)?; + if let Some(existing) = object.get("table-uuid").and_then(serde_json::Value::as_str) + && existing != uuid + { + return Err(s3_error!(PreconditionFailed, "cannot reassign table uuid")); + } + object.insert("table-uuid".to_string(), serde_json::Value::String(uuid.to_string())); + Ok(()) +} + +fn apply_add_view_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let mut view_version = update + .get("view-version") + .cloned() + .ok_or_else(|| s3_error!(InvalidRequest, "add-view-version requires view-version"))?; + if !view_version.is_object() { + return Err(s3_error!(InvalidRequest, "view-version must be a JSON object")); + } + if view_version.get("version-id").is_none() { + let next_id = next_array_object_i64(metadata, "versions", "version-id")?; + view_version + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))? + .insert("version-id".to_string(), serde_json::Value::from(next_id)); + } + view_version + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))? + .entry("timestamp-ms".to_string()) + .or_insert_with(|| serde_json::Value::from(current_time_millis())); + ensure_array_field(metadata, "versions")?.push(view_version); + Ok(()) +} + +fn apply_set_current_view_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let requested_id = update + .get("view-version-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "set-current-view-version requires view-version-id"))?; + let version_id = if requested_id == -1 { + last_array_object_i64(metadata, "versions", "version-id")? + } else { + requested_id + }; + metadata_object_mut(metadata)?.insert("current-version-id".to_string(), serde_json::Value::from(version_id)); + ensure_array_field(metadata, "version-log")?.push(serde_json::json!({ + "timestamp-ms": current_time_millis(), + "version-id": version_id + })); + Ok(()) +} + +fn apply_upgrade_format_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let version = update + .get("format-version") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "upgrade-format-version requires format-version"))?; + let current = metadata + .get("format-version") + .and_then(serde_json::Value::as_i64) + .unwrap_or_default(); + if version < current { + return Err(s3_error!(InvalidRequest, "format-version cannot be downgraded")); + } + metadata_object_mut(metadata)?.insert("format-version".to_string(), serde_json::Value::from(version)); + Ok(()) +} + +fn apply_add_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let mut schema = update + .get("schema") + .cloned() + .ok_or_else(|| s3_error!(InvalidRequest, "add-schema requires schema"))?; + if !schema.is_object() { + return Err(s3_error!(InvalidRequest, "add-schema schema must be a JSON object")); + } + if schema.get("schema-id").is_none() { + let next_id = next_array_object_i64(metadata, "schemas", "schema-id")?; + schema + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-schema schema must be a JSON object"))? + .insert("schema-id".to_string(), serde_json::Value::from(next_id)); + } + let last_column_id = max_field_id(&schema); + ensure_array_field(metadata, "schemas")?.push(schema); + let object = metadata_object_mut(metadata)?; + let current_last = object + .get("last-column-id") + .and_then(serde_json::Value::as_i64) + .unwrap_or_default(); + object.insert("last-column-id".to_string(), serde_json::Value::from(current_last.max(last_column_id))); + Ok(()) +} + +fn apply_set_current_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let requested_id = update + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "set-current-schema requires schema-id"))?; + let schema_id = if requested_id == -1 { + last_array_object_i64(metadata, "schemas", "schema-id")? + } else { + requested_id + }; + metadata_object_mut(metadata)?.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); + Ok(()) +} + +fn apply_add_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let mut spec = update + .get("spec") + .cloned() + .ok_or_else(|| s3_error!(InvalidRequest, "add-spec requires spec"))?; + if !spec.is_object() { + return Err(s3_error!(InvalidRequest, "add-spec spec must be a JSON object")); + } + if spec.get("spec-id").is_none() { + let next_id = next_array_object_i64(metadata, "partition-specs", "spec-id")?; + spec.as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-spec spec must be a JSON object"))? + .insert("spec-id".to_string(), serde_json::Value::from(next_id)); + } + let last_partition_id = max_partition_field_id(&spec); + ensure_array_field(metadata, "partition-specs")?.push(spec); + let object = metadata_object_mut(metadata)?; + let current_last = object + .get("last-partition-id") + .and_then(serde_json::Value::as_i64) + .unwrap_or(999); + object.insert( + "last-partition-id".to_string(), + serde_json::Value::from(current_last.max(last_partition_id)), + ); + Ok(()) +} + +fn apply_set_default_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let requested_id = update + .get("spec-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "set-default-spec requires spec-id"))?; + let spec_id = if requested_id == -1 { + last_array_object_i64(metadata, "partition-specs", "spec-id")? + } else { + requested_id + }; + metadata_object_mut(metadata)?.insert("default-spec-id".to_string(), serde_json::Value::from(spec_id)); + Ok(()) +} + +fn apply_add_sort_order_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let mut sort_order = update + .get("sort-order") + .cloned() + .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order requires sort-order"))?; + if !sort_order.is_object() { + return Err(s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object")); + } + if sort_order.get("order-id").is_none() { + let next_id = next_array_object_i64(metadata, "sort-orders", "order-id")?; + sort_order + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object"))? + .insert("order-id".to_string(), serde_json::Value::from(next_id)); + } + ensure_array_field(metadata, "sort-orders")?.push(sort_order); + Ok(()) +} + +fn apply_set_default_sort_order_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let requested_id = update + .get("sort-order-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "set-default-sort-order requires sort-order-id"))?; + let sort_order_id = if requested_id == -1 { + last_array_object_i64(metadata, "sort-orders", "order-id")? + } else { + requested_id + }; + metadata_object_mut(metadata)?.insert("default-sort-order-id".to_string(), serde_json::Value::from(sort_order_id)); + Ok(()) +} + +fn apply_add_snapshot_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let snapshot = update + .get("snapshot") + .cloned() + .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?; + let snapshot_id = snapshot + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-id must be an integer"))?; + let sequence_number = snapshot + .get("sequence-number") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer"))?; + let timestamp_ms = snapshot + .get("timestamp-ms") + .and_then(serde_json::Value::as_i64) + .unwrap_or_else(current_time_millis); + validate_added_snapshot(metadata, &snapshot, snapshot_id, sequence_number)?; + ensure_array_field(metadata, "snapshots")?.push(snapshot); + let object = metadata_object_mut(metadata)?; + object.insert("last-sequence-number".to_string(), serde_json::Value::from(sequence_number)); + object.insert("current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id)); + ensure_array_field(metadata, "snapshot-log")?.push(serde_json::json!({ + "timestamp-ms": timestamp_ms, + "snapshot-id": snapshot_id + })); + Ok(()) +} + +fn validate_added_snapshot( + metadata: &serde_json::Value, + snapshot: &serde_json::Value, + snapshot_id: i64, + sequence_number: i64, +) -> S3Result<()> { + if metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .is_some_and(|snapshots| { + snapshots + .iter() + .any(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) + }) + { + return Err(s3_error!(PreconditionFailed, "snapshot id already exists")); + } + + let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); + if let Some(parent_snapshot_id) = snapshot.get("parent-snapshot-id").and_then(serde_json::Value::as_i64) + && Some(parent_snapshot_id) != current_snapshot_id + { + return Err(s3_error!(PreconditionFailed, "snapshot parent no longer matches current snapshot")); + } + + let current_sequence_number = metadata + .get("last-sequence-number") + .and_then(serde_json::Value::as_i64) + .unwrap_or_default(); + if sequence_number <= current_sequence_number { + return Err(s3_error!(PreconditionFailed, "snapshot sequence number must advance")); + } + + if !snapshot_has_manifest_references(snapshot) { + return Err(s3_error!(InvalidRequest, "snapshot manifest-list or manifests are required")); + } + + let operation = snapshot + .get("summary") + .and_then(|summary| summary.get("operation")) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot summary.operation is required"))?; + if !matches!(operation, "append" | "overwrite" | "delete" | "replace") { + return Err(s3_error!(NotImplemented, "unsupported snapshot operation: {operation}")); + } + + Ok(()) +} + +fn snapshot_has_manifest_references(snapshot: &serde_json::Value) -> bool { + if snapshot + .get("manifest-list") + .and_then(serde_json::Value::as_str) + .is_some_and(|manifest_list| !manifest_list.is_empty()) + { + return true; + } + snapshot + .get("manifests") + .and_then(serde_json::Value::as_array) + .is_some_and(|manifests| { + !manifests.is_empty() + && manifests + .iter() + .all(|manifest| manifest.as_str().is_some_and(|manifest| !manifest.is_empty())) + }) +} + +#[derive(Default)] +struct SnapshotLiveFiles { + data_files: BTreeMap, + delete_files: BTreeMap, + manifest_files: BTreeMap, +} + +impl SnapshotLiveFiles { + fn contains(&self, location: &str) -> bool { + self.data_files.contains_key(location) || self.delete_files.contains_key(location) + } + + fn identity( + &self, + location: &str, + object_kind: &crate::table_catalog::TableMetadataMaintenanceObjectKind, + ) -> Option<&SnapshotFileIdentity> { + match object_kind { + crate::table_catalog::TableMetadataMaintenanceObjectKind::DataFile => self.data_files.get(location), + crate::table_catalog::TableMetadataMaintenanceObjectKind::DeleteFile => self.delete_files.get(location), + _ => None, + } + } +} + +#[derive(Default)] +struct SnapshotFileChanges { + added_data_files: BTreeSet, + added_delete_files: BTreeSet, + deleted_data_files: BTreeSet, + deleted_delete_files: BTreeSet, +} + +impl SnapshotFileChanges { + fn has_delete_or_row_level_change(&self) -> bool { + !self.added_delete_files.is_empty() || !self.deleted_data_files.is_empty() || !self.deleted_delete_files.is_empty() + } + + fn has_any_change(&self) -> bool { + !self.added_data_files.is_empty() || !self.added_delete_files.is_empty() || self.has_deleted_files() + } + + fn has_deleted_files(&self) -> bool { + !self.deleted_data_files.is_empty() || !self.deleted_delete_files.is_empty() + } +} + +struct SnapshotChangeContext<'a> { + current_live_files: &'a SnapshotLiveFiles, + snapshot_id: i64, + sequence_number: i64, +} + +#[derive(PartialEq, Eq)] +struct SnapshotManifestIdentity { + sequence_number: Option, + added_snapshot_id: Option, +} + +#[derive(PartialEq, Eq)] +struct SnapshotFileIdentity { + sequence_number: Option, + file_sequence_number: Option, +} + +async fn validate_table_snapshot_commit_conflicts( + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &crate::table_catalog::IdentifierSegment, + entry: &crate::table_catalog::TableEntry, + current_metadata: &serde_json::Value, + updates: &[serde_json::Value], +) -> S3Result<()> +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let Some(snapshot) = added_snapshot_update(updates)? else { + return Ok(()); + }; + let snapshot_id = snapshot + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-id must be an integer"))?; + let sequence_number = snapshot + .get("sequence-number") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer"))?; + let operation = snapshot + .get("summary") + .and_then(|summary| summary.get("operation")) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot summary.operation is required"))?; + + let current_live_files = + load_current_snapshot_live_files(metadata_backend, bucket, namespace, table, entry, current_metadata).await?; + let changes = load_snapshot_file_changes( + metadata_backend, + bucket, + namespace, + table, + entry, + snapshot, + SnapshotChangeContext { + current_live_files: ¤t_live_files, + snapshot_id, + sequence_number, + }, + ) + .await?; + + for location in changes.added_data_files.iter().chain(changes.added_delete_files.iter()) { + if current_live_files.contains(location) { + return Err(s3_error!( + PreconditionFailed, + "commit requirement failed: added file already exists in current snapshot" + )); + } + } + + match operation { + "append" => { + if changes.has_deleted_files() || !changes.added_delete_files.is_empty() { + return Err(s3_error!(InvalidRequest, "append snapshot cannot delete data files or add delete files")); + } + } + "overwrite" | "delete" | "replace" => { + if current_metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_none() + { + return Err(s3_error!(InvalidRequest, "row-level snapshot operation requires a current snapshot")); + } + if operation == "overwrite" { + if !changes.has_any_change() { + return Err(s3_error!(InvalidRequest, "overwrite snapshot operation requires changed files")); + } + } else if !changes.has_delete_or_row_level_change() { + return Err(s3_error!( + InvalidRequest, + "row-level snapshot operation requires deleted data files or added delete files" + )); + } + for location in changes.deleted_data_files.iter().chain(changes.deleted_delete_files.iter()) { + if !current_live_files.contains(location) { + return Err(s3_error!(PreconditionFailed, "commit requirement failed: deleted file is not current")); + } + } + } + _ => return Err(s3_error!(NotImplemented, "unsupported snapshot operation: {operation}")), + } + + Ok(()) +} + +fn added_snapshot_update(updates: &[serde_json::Value]) -> S3Result> { + let mut snapshot = None; + for update in updates { + if update.get("action").and_then(serde_json::Value::as_str) != Some("add-snapshot") { + continue; + } + if snapshot.is_some() { + return Err(s3_error!(InvalidRequest, "standard commit supports one add-snapshot update")); + } + snapshot = Some( + update + .get("snapshot") + .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?, + ); + } + Ok(snapshot) +} + +async fn load_current_snapshot_live_files( + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &crate::table_catalog::IdentifierSegment, + entry: &crate::table_catalog::TableEntry, + current_metadata: &serde_json::Value, +) -> S3Result +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let Some(current_snapshot_id) = current_metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64) + else { + return Ok(SnapshotLiveFiles::default()); + }; + let snapshot = current_metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .and_then(|snapshots| { + snapshots + .iter() + .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(current_snapshot_id)) + }) + .ok_or_else(|| s3_error!(InvalidRequest, "current snapshot metadata is missing"))?; + + let mut live_files = SnapshotLiveFiles::default(); + for manifest in read_snapshot_manifest_references(metadata_backend, bucket, namespace, table, entry, snapshot).await? { + let SnapshotManifestLocation { + manifest_path, + sequence_number, + added_snapshot_id, + } = manifest.location; + live_files.manifest_files.insert( + manifest_path, + SnapshotManifestIdentity { + sequence_number, + added_snapshot_id, + }, + ); + for reference in manifest.references { + let status = reference + .entry_status + .ok_or_else(|| s3_error!(InvalidRequest, "manifest entry status is required"))?; + match status { + 0 | 1 => { + let identity = SnapshotFileIdentity { + sequence_number: reference.sequence_number, + file_sequence_number: reference.file_sequence_number, + }; + match reference.object_kind { + crate::table_catalog::TableMetadataMaintenanceObjectKind::DataFile => { + live_files.data_files.insert(reference.location, identity); + } + crate::table_catalog::TableMetadataMaintenanceObjectKind::DeleteFile => { + live_files.delete_files.insert(reference.location, identity); + } + _ => {} + } + } + 2 => {} + _ => return Err(s3_error!(InvalidRequest, "manifest entry status is unsupported")), + } + } + } + Ok(live_files) +} + +async fn load_snapshot_file_changes( + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &crate::table_catalog::IdentifierSegment, + entry: &crate::table_catalog::TableEntry, + snapshot: &serde_json::Value, + context: SnapshotChangeContext<'_>, +) -> S3Result +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let mut changes = SnapshotFileChanges::default(); + for manifest in read_snapshot_manifest_references(metadata_backend, bucket, namespace, table, entry, snapshot).await? { + let inherited_identity = context + .current_live_files + .manifest_files + .get(&manifest.location.manifest_path); + if let Some(inherited_identity) = inherited_identity { + let candidate_identity = SnapshotManifestIdentity { + sequence_number: manifest.location.sequence_number, + added_snapshot_id: manifest.location.added_snapshot_id, + }; + if inherited_identity + .sequence_number + .is_some_and(|sequence_number| candidate_identity.sequence_number != Some(sequence_number)) + || inherited_identity + .added_snapshot_id + .is_some_and(|snapshot_id| candidate_identity.added_snapshot_id != Some(snapshot_id)) + { + return Err(s3_error!(InvalidRequest, "inherited manifest must preserve its manifest-list identity")); + } + continue; + } + if manifest + .location + .added_snapshot_id + .is_some_and(|added_snapshot_id| added_snapshot_id != context.snapshot_id) + { + return Err(s3_error!(InvalidRequest, "new manifest must belong to the committed snapshot")); + } + if manifest + .location + .sequence_number + .is_some_and(|sequence_number| sequence_number != context.sequence_number) + { + return Err(s3_error!(InvalidRequest, "new manifest sequence must match the committed snapshot")); + } + + for reference in manifest.references { + let status = reference + .entry_status + .ok_or_else(|| s3_error!(InvalidRequest, "manifest entry status is required"))?; + if matches!(status, 1 | 2) && reference.snapshot_id != Some(context.snapshot_id) { + return Err(s3_error!( + InvalidRequest, + "manifest changed entries must belong to the committed snapshot" + )); + } + if status == 1 + && manifest.location.sequence_number.is_some_and(|sequence_number| { + reference.sequence_number != Some(sequence_number) || reference.file_sequence_number != Some(sequence_number) + }) + { + return Err(s3_error!(InvalidRequest, "added manifest entry sequence must match its manifest")); + } + if status == 2 + && context + .current_live_files + .identity(&reference.location, &reference.object_kind) + .is_some_and(|current_identity| { + current_identity + != &SnapshotFileIdentity { + sequence_number: reference.sequence_number, + file_sequence_number: reference.file_sequence_number, + } + }) + { + return Err(s3_error!( + InvalidRequest, + "deleted manifest entry must preserve the current file sequence" + )); + } + + match (status, reference.object_kind) { + (0, _) => {} + (1, crate::table_catalog::TableMetadataMaintenanceObjectKind::DataFile) => { + changes.added_data_files.insert(reference.location); + } + (1, crate::table_catalog::TableMetadataMaintenanceObjectKind::DeleteFile) => { + changes.added_delete_files.insert(reference.location); + } + (2, crate::table_catalog::TableMetadataMaintenanceObjectKind::DataFile) => { + changes.deleted_data_files.insert(reference.location); + } + (2, crate::table_catalog::TableMetadataMaintenanceObjectKind::DeleteFile) => { + changes.deleted_delete_files.insert(reference.location); + } + _ => return Err(s3_error!(InvalidRequest, "manifest entry status is unsupported")), + } + } + } + Ok(changes) +} + +struct SnapshotManifestReferences { + location: SnapshotManifestLocation, + references: Vec, +} + +async fn read_snapshot_manifest_references( + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &crate::table_catalog::IdentifierSegment, + entry: &crate::table_catalog::TableEntry, + snapshot: &serde_json::Value, +) -> S3Result> +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let manifest_locations = snapshot_manifest_locations(metadata_backend, bucket, namespace, table, entry, snapshot).await?; + let mut manifests = Vec::new(); + for manifest_location in manifest_locations { + let manifest_key = table_commit_object_key( + bucket, + namespace, + table, + entry, + &manifest_location.manifest_path, + crate::table_catalog::TableMetadataMaintenanceObjectKind::ManifestFile, + )?; + let manifest_object = metadata_backend + .read_object(bucket, &manifest_key) + .await + .map_err(catalog_store_error)? + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot manifest object is missing"))?; + let file_references = + crate::table_catalog::data_file_references_from_manifest_avro(&manifest_object.data).map_err(catalog_store_error)?; + let mut references = Vec::with_capacity(file_references.len()); + for mut reference in file_references { + if reference.snapshot_id.is_none() { + reference.snapshot_id = manifest_location.added_snapshot_id; + } + if reference.sequence_number.is_none() { + reference.sequence_number = manifest_location.sequence_number; + } + if reference.file_sequence_number.is_none() { + reference.file_sequence_number = manifest_location.sequence_number; + } + validate_manifest_data_file_reference(metadata_backend, bucket, namespace, table, entry, &reference).await?; + references.push(reference); + } + manifests.push(SnapshotManifestReferences { + location: manifest_location, + references, + }); + } + Ok(manifests) +} + +#[derive(Debug)] +struct SnapshotManifestLocation { + manifest_path: String, + sequence_number: Option, + added_snapshot_id: Option, +} + +async fn snapshot_manifest_locations( + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &crate::table_catalog::IdentifierSegment, + entry: &crate::table_catalog::TableEntry, + snapshot: &serde_json::Value, +) -> S3Result> +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + if let Some(manifest_list_location) = snapshot.get("manifest-list").and_then(serde_json::Value::as_str) { + let manifest_list_key = table_commit_object_key( + bucket, + namespace, + table, + entry, + manifest_list_location, + crate::table_catalog::TableMetadataMaintenanceObjectKind::ManifestList, + )?; + let manifest_list_object = metadata_backend + .read_object(bucket, &manifest_list_key) + .await + .map_err(catalog_store_error)? + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot manifest-list object is missing"))?; + let references = crate::table_catalog::manifest_list_references_from_manifest_list_avro(&manifest_list_object.data) + .map_err(catalog_store_error)?; + if references.is_empty() { + return Err(s3_error!(InvalidRequest, "snapshot manifest-list must reference at least one manifest")); + } + return Ok(references + .into_iter() + .map(|reference| SnapshotManifestLocation { + manifest_path: reference.manifest_path, + sequence_number: reference.sequence_number, + added_snapshot_id: reference.added_snapshot_id, + }) + .collect()); + } + + let Some(manifests) = snapshot.get("manifests").and_then(serde_json::Value::as_array) else { + return Err(s3_error!(InvalidRequest, "snapshot manifest-list is required")); + }; + if manifests.is_empty() { + return Err(s3_error!(InvalidRequest, "snapshot manifests must reference at least one manifest")); + } + manifests + .iter() + .map(|manifest| { + manifest + .as_str() + .filter(|manifest| !manifest.is_empty()) + .map(|manifest| SnapshotManifestLocation { + manifest_path: manifest.to_string(), + sequence_number: None, + added_snapshot_id: None, + }) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot manifest location must be a string")) + }) + .collect() +} + +async fn validate_manifest_data_file_reference( + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &crate::table_catalog::IdentifierSegment, + entry: &crate::table_catalog::TableEntry, + reference: &crate::table_catalog::ManifestDataFileReference, +) -> S3Result<()> +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + table_commit_object_key(bucket, namespace, table, entry, &reference.location, reference.object_kind.clone())?; + let object_key = crate::table_catalog::table_catalog_object_key_from_location(bucket, &reference.location) + .ok_or_else(|| s3_error!(InvalidRequest, "manifest data file location is invalid"))?; + if !metadata_backend + .object_exists(bucket, &object_key) + .await + .map_err(catalog_store_error)? + { + return Err(s3_error!(InvalidRequest, "manifest referenced data file is missing")); + } + Ok(()) +} + +fn table_commit_object_key( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &crate::table_catalog::IdentifierSegment, + entry: &crate::table_catalog::TableEntry, + location: &str, + expected_kind: crate::table_catalog::TableMetadataMaintenanceObjectKind, +) -> S3Result { + let object_key = crate::table_catalog::table_catalog_object_key_from_location(bucket, location) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot object location is invalid"))?; + let warehouse_object_prefix = crate::table_catalog::table_warehouse_object_prefix(entry).map_err(catalog_store_error)?; + let object_kind = + crate::table_catalog::table_maintenance_object_kind(namespace, table, Some(&warehouse_object_prefix), &object_key) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot object is outside the table warehouse"))?; + if object_kind != expected_kind { + return Err(s3_error!(InvalidRequest, "snapshot object kind does not match manifest metadata")); + } + Ok(object_key) +} + +fn apply_set_snapshot_ref_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let ref_name = update + .get("ref-name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "set-snapshot-ref requires ref-name"))?; + let snapshot_id = update + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "set-snapshot-ref requires snapshot-id"))?; + let reference = update + .as_object() + .ok_or_else(|| s3_error!(InvalidRequest, "set-snapshot-ref must be a JSON object"))? + .iter() + .filter(|(key, _)| key.as_str() != "action" && key.as_str() != "ref-name") + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + ensure_object_field(metadata, "refs")?.insert(ref_name.to_string(), serde_json::Value::Object(reference)); + if ref_name == "main" { + metadata_object_mut(metadata)?.insert("current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id)); + } + Ok(()) +} + +fn apply_remove_snapshots_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let ids = update + .get("snapshot-ids") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| s3_error!(InvalidRequest, "remove-snapshots requires snapshot-ids"))? + .iter() + .filter_map(serde_json::Value::as_i64) + .collect::>(); + ensure_array_field(metadata, "snapshots")?.retain(|snapshot| { + snapshot + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) + }); + ensure_array_field(metadata, "snapshot-log")?.retain(|log| { + log.get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) + }); + Ok(()) +} + +fn apply_remove_snapshot_ref_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let ref_name = update + .get("ref-name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "remove-snapshot-ref requires ref-name"))?; + ensure_object_field(metadata, "refs")?.remove(ref_name); + Ok(()) +} + +fn apply_set_location_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let location = update + .get("location") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "set-location requires location"))?; + metadata_object_mut(metadata)?.insert("location".to_string(), serde_json::Value::String(location.to_string())); + Ok(()) +} + +fn apply_set_properties_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let updates = update + .get("updates") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| s3_error!(InvalidRequest, "set-properties requires updates"))?; + let properties = ensure_object_field(metadata, "properties")?; + for (key, value) in updates { + let value = value + .as_str() + .ok_or_else(|| s3_error!(InvalidRequest, "table property values must be strings"))?; + properties.insert(key.clone(), serde_json::Value::String(value.to_string())); + } + Ok(()) +} + +fn apply_remove_properties_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { + let removals = update + .get("removals") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| s3_error!(InvalidRequest, "remove-properties requires removals"))?; + let properties = ensure_object_field(metadata, "properties")?; + for removal in removals { + let key = removal + .as_str() + .ok_or_else(|| s3_error!(InvalidRequest, "property removals must be strings"))?; + properties.remove(key); + } + Ok(()) +} + +fn append_previous_metadata_log(metadata: &mut serde_json::Value, previous_metadata_location: &str) -> S3Result<()> { + ensure_array_field(metadata, "metadata-log")?.push(serde_json::json!({ + "timestamp-ms": current_time_millis(), + "metadata-file": previous_metadata_location + })); + Ok(()) +} + +fn metadata_object_mut(metadata: &mut serde_json::Value) -> S3Result<&mut serde_json::Map> { + metadata + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "table metadata must be a JSON object")) +} + +fn ensure_array_field<'a>(metadata: &'a mut serde_json::Value, key: &str) -> S3Result<&'a mut Vec> { + let object = metadata_object_mut(metadata)?; + object + .entry(key.to_string()) + .or_insert_with(|| serde_json::Value::Array(Vec::new())); + object + .get_mut(key) + .and_then(serde_json::Value::as_array_mut) + .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {key} must be an array")) +} + +fn ensure_object_field<'a>( + metadata: &'a mut serde_json::Value, + key: &str, +) -> S3Result<&'a mut serde_json::Map> { + let object = metadata_object_mut(metadata)?; + object + .entry(key.to_string()) + .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new())); + object + .get_mut(key) + .and_then(serde_json::Value::as_object_mut) + .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {key} must be an object")) +} + +fn next_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result { + Ok(last_array_object_i64(metadata, array_key, id_key)?.saturating_add(1)) +} + +fn last_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result { + let values = metadata + .get(array_key) + .and_then(serde_json::Value::as_array) + .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} must be an array"))?; + values + .iter() + .filter_map(|value| value.get(id_key).and_then(serde_json::Value::as_i64)) + .max() + .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} has no {id_key}")) +} + +fn table_commit_operation(metadata: &serde_json::Value) -> String { + metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .and_then(|snapshots| snapshots.last()) + .and_then(|snapshot| snapshot.get("summary")) + .and_then(|summary| summary.get("operation")) + .and_then(serde_json::Value::as_str) + .unwrap_or("commit") + .to_string() +} + +fn namespace_entry_from_create_request( + bucket: &str, + request: CreateNamespaceRequest, +) -> S3Result { + let namespace = namespace_from_segments(&request.namespace)?; + Ok(crate::table_catalog::NamespaceEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + namespace_id: namespace.storage_id(), + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: request.properties, + created_at: None, + updated_at: None, + }) +} + +fn iceberg_rest_error(error_type: &str, status: StatusCode, message: impl Into) -> S3Error { + let mut err = S3Error::with_message(S3ErrorCode::Custom(error_type.into()), message.into()); + err.set_status_code(status); + err +} + +fn catalog_store_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { + match err { + crate::table_catalog::TableCatalogStoreError::NotFound(message) => { + iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_RESOURCE, StatusCode::NOT_FOUND, message) + } + crate::table_catalog::TableCatalogStoreError::Conflict(message) => { + iceberg_rest_error(ICEBERG_ERROR_COMMIT_FAILED, StatusCode::CONFLICT, message) + } + crate::table_catalog::TableCatalogStoreError::Invalid(message) => { + iceberg_rest_error(ICEBERG_ERROR_BAD_REQUEST, StatusCode::BAD_REQUEST, message) + } + crate::table_catalog::TableCatalogStoreError::Internal(message) => { + iceberg_rest_error(ICEBERG_ERROR_REST, StatusCode::INTERNAL_SERVER_ERROR, message) + } + } +} + +fn catalog_store_conflict_error(err: crate::table_catalog::TableCatalogStoreError, conflict_type: &'static str) -> S3Error { + match err { + crate::table_catalog::TableCatalogStoreError::Conflict(message) => { + iceberg_rest_error(conflict_type, StatusCode::CONFLICT, message) + } + err => catalog_store_error(err), + } +} + +fn catalog_store_already_exists_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { + catalog_store_conflict_error(err, ICEBERG_ERROR_ALREADY_EXISTS) +} + +fn catalog_store_namespace_drop_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { + match err { + crate::table_catalog::TableCatalogStoreError::NotFound(message) => { + iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_NAMESPACE, StatusCode::NOT_FOUND, message) + } + err => catalog_store_conflict_error(err, ICEBERG_ERROR_NAMESPACE_NOT_EMPTY), + } +} + +async fn create_namespace_response( + store: &S, + bucket: &str, + request: CreateNamespaceRequest, + table_bucket_enabled: bool, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let entry = namespace_entry_from_create_request(bucket, request)?; + ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + store + .create_namespace(entry.clone()) + .await + .map_err(catalog_store_already_exists_error)?; + namespace_response_from_entry(entry) +} + +async fn list_namespaces_response(store: &S, bucket: &str, uri: &http::Uri) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let context = RestPageContext { + resource: TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT, + warehouse: bucket, + namespace: None, + }; + let pagination = rest_pagination_from_query(uri, context)?; + let page = match pagination.page_request() { + Some((cursor, limit)) => store + .list_namespaces_page(bucket, cursor, limit) + .await + .map_err(catalog_store_error)?, + None => crate::table_catalog::TableCatalogListPage { + entries: store.list_namespaces(bucket).await.map_err(catalog_store_error)?, + next_cursor: None, + }, + }; + let next_page_token = pagination.next_page_token(page.next_cursor)?; + list_namespaces_response_from_entries(page.entries, next_page_token) +} + +async fn get_namespace_response( + store: &S, + bucket: &str, + namespace: &crate::table_catalog::Namespace, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(entry) = store + .get_namespace(bucket, &namespace.public_name()) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error( + ICEBERG_ERROR_NO_SUCH_NAMESPACE, + StatusCode::NOT_FOUND, + "namespace not found", + )); + }; + namespace_response_from_entry(entry) +} + +async fn namespace_exists_status(store: &S, bucket: &str, namespace: &crate::table_catalog::Namespace) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let exists = store + .get_namespace(bucket, &namespace.public_name()) + .await + .map_err(catalog_store_error)? + .is_some(); + Ok(exists_status(exists)) +} + +async fn drop_namespace_in_store(store: &S, bucket: &str, namespace: &str) -> S3Result<()> +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + store + .drop_namespace(bucket, namespace) + .await + .map_err(catalog_store_namespace_drop_error) +} + +async fn register_table_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + request: RegisterTableRequest, + table_bucket_enabled: bool, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let mut entry = table_entry_from_register_request(bucket, namespace, request)?; + ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &metadata)?; + adopt_registered_metadata_identity(&mut entry, &metadata)?; + store + .register_table(entry.clone()) + .await + .map_err(catalog_store_already_exists_error)?; + Ok(load_table_response_from_entry(entry, metadata)) +} + +async fn create_table_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + request: CreateTableRequest, + table_bucket_enabled: bool, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let (entry, metadata) = table_entry_from_create_table_request(bucket, namespace, request)?; + ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + let metadata_data = serde_json::to_vec(&metadata) + .map_err(|err| s3_error!(InternalError, "failed to serialize initial table metadata: {}", err))?; + metadata_backend + .put_object( + bucket, + &entry.metadata_location, + metadata_data, + crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, + ) + .await + .map_err(catalog_store_already_exists_error)?; + store + .create_table(entry.clone()) + .await + .map_err(catalog_store_already_exists_error)?; + Ok(load_table_response_from_entry(entry, metadata)) +} + +async fn create_view_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + request: CreateViewRequest, + table_bucket_enabled: bool, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let (entry, metadata) = view_entry_from_create_view_request(bucket, namespace, request)?; + ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + let metadata_data = serde_json::to_vec(&metadata) + .map_err(|err| s3_error!(InternalError, "failed to serialize initial view metadata: {}", err))?; + metadata_backend + .put_object( + bucket, + &entry.metadata_location, + metadata_data, + crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, + ) + .await + .map_err(catalog_store_already_exists_error)?; + store + .create_view(entry.clone()) + .await + .map_err(catalog_store_already_exists_error)?; + Ok(load_view_response_from_entry(entry, metadata)) +} + +async fn read_table_metadata_json( + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + metadata_location: &str, +) -> S3Result { + let Some(object) = metadata_backend + .read_object(bucket, metadata_location) + .await + .map_err(catalog_store_error)? + else { + return Err(s3_error!(InvalidRequest, "table metadata object not found: {metadata_location}")); + }; + let metadata = serde_json::from_slice::(&object.data) + .map_err(|err| s3_error!(InvalidRequest, "failed to parse table metadata JSON: {}", err))?; + if !metadata.is_object() { + return Err(s3_error!(InvalidRequest, "table metadata JSON must be an object")); + } + Ok(metadata) +} + +async fn list_tables_response( + store: &S, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + uri: &http::Uri, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let namespace = namespace.public_name(); + let context = RestPageContext { + resource: TABLE_CATALOG_TABLE_RESOURCE_ROOT, + warehouse: bucket, + namespace: Some(&namespace), + }; + let pagination = rest_pagination_from_query(uri, context)?; + let page = match pagination.page_request() { + Some((cursor, limit)) => store + .list_tables_page(bucket, &namespace, cursor, limit) + .await + .map_err(catalog_store_error)?, + None => crate::table_catalog::TableCatalogListPage { + entries: store.list_tables(bucket, &namespace).await.map_err(catalog_store_error)?, + next_cursor: None, + }, + }; + let next_page_token = pagination.next_page_token(page.next_cursor)?; + list_tables_response_from_entries(page.entries, next_page_token) +} + +async fn load_table_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(entry) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + Ok(load_table_response_from_entry(entry, metadata)) +} + +async fn list_views_response( + store: &S, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + uri: &http::Uri, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let namespace = namespace.public_name(); + let context = RestPageContext { + resource: TABLE_CATALOG_VIEW_RESOURCE_ROOT, + warehouse: bucket, + namespace: Some(&namespace), + }; + let pagination = rest_pagination_from_query(uri, context)?; + let page = match pagination.page_request() { + Some((cursor, limit)) => store + .list_views_page(bucket, &namespace, cursor, limit) + .await + .map_err(catalog_store_error)?, + None => crate::table_catalog::TableCatalogListPage { + entries: store.list_views(bucket, &namespace).await.map_err(catalog_store_error)?, + next_cursor: None, + }, + }; + let next_page_token = pagination.next_page_token(page.next_cursor)?; + list_views_response_from_entries(page.entries, next_page_token) +} + +async fn load_view_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + view: &str, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(entry) = store + .load_view(bucket, &namespace.public_name(), view) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_VIEW, StatusCode::NOT_FOUND, "view not found")); + }; + let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + Ok(load_view_response_from_entry(entry, metadata)) +} + +async fn view_exists_status( + store: &S, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + view: &str, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let exists = store + .load_view(bucket, &namespace.public_name(), view) + .await + .map_err(catalog_store_error)? + .is_some(); + Ok(exists_status(exists)) +} + +async fn replace_view_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + view: &str, + request: RestCommitViewRequest, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(current) = store + .load_view(bucket, &namespace.public_name(), view) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_VIEW, StatusCode::NOT_FOUND, "view not found")); + }; + let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + validate_view_commit_requirements(¤t_metadata, &request.requirements)?; + let view_name = crate::table_catalog::IdentifierSegment::parse(view.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid view name: {}", err))?; + let (next_metadata_location, next_metadata) = if let Some(new_metadata_location) = request.new_metadata_location { + if !crate::table_catalog::is_valid_view_metadata_location(namespace, &view_name, &new_metadata_location) { + return Err(s3_error!(InvalidRequest, "metadata location must be inside the view metadata directory")); + } + let target_metadata = read_table_metadata_json(metadata_backend, bucket, &new_metadata_location).await?; + validate_metadata_view_location_in_bucket(bucket, &target_metadata)?; + validate_metadata_matches_current_view_metadata(¤t_metadata, &target_metadata)?; + (new_metadata_location, target_metadata) + } else { + let next_metadata = apply_view_commit_updates(current_metadata.clone(), &request.updates, ¤t.metadata_location)?; + validate_metadata_view_location_in_bucket(bucket, &next_metadata)?; + validate_metadata_matches_current_view_metadata(¤t_metadata, &next_metadata)?; + let (_, metadata_file_token) = standard_commit_ids(request.commit_id); + let next_generation = current.generation.saturating_add(1); + let next_metadata_location = crate::table_catalog::default_view_metadata_file_path( + namespace, + &view_name, + &next_metadata_file_name(next_generation, &metadata_file_token), + ); + let next_metadata_data = serde_json::to_vec(&next_metadata) + .map_err(|err| s3_error!(InternalError, "failed to serialize view metadata update: {}", err))?; + metadata_backend + .put_object( + bucket, + &next_metadata_location, + next_metadata_data, + crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, + ) + .await + .map_err(catalog_store_error)?; + (next_metadata_location, next_metadata) + }; + + let result = store + .replace_view(crate::table_catalog::ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.to_string(), + expected_version_token: request + .expected_version_token + .unwrap_or_else(|| current.version_token.clone()), + expected_metadata_location: request + .expected_metadata_location + .unwrap_or_else(|| current.metadata_location.clone()), + new_metadata_location: next_metadata_location, + }) + .await + .map_err(catalog_store_error)?; + Ok(load_view_response_from_entry(result.view, next_metadata)) +} + +async fn table_exists_status( + store: &S, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let exists = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + .is_some(); + Ok(exists_status(exists)) +} + +async fn load_credentials_response( + store: &S, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + issuer: &dyn TableCredentialIssuer, + principal: Option<&rustfs_credentials::Credentials>, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(entry) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + load_credentials_response_from_entry(&entry, issuer, principal).await +} + +async fn get_table_metadata_location_response( + store: &S, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(entry) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + Ok(table_metadata_location_response_from_entry(entry)) +} + +async fn update_table_metadata_location_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: UpdateTableMetadataLocationRequest, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(current) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; + if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &metadata_location) { + return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); + } + let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; + validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; + let commit_request = crate::table_catalog::TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + commit_id: request.commit_id.unwrap_or_else(|| Uuid::new_v4().to_string()), + idempotency_key: request.idempotency_key, + operation: "update-metadata-location".to_string(), + expected_version_token: request.version_token, + expected_metadata_location: current.metadata_location, + new_metadata_location: metadata_location, + requirements: Vec::new(), + writer: Some("rustfs-metadata-location-api".to_string()), + }; + let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; + Ok(table_metadata_location_response_from_entry(result.table)) +} + +async fn commit_table_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: RestCommitTableRequest, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + if request.new_metadata_location.is_none() { + return standard_commit_table_response(store, metadata_backend, bucket, namespace, table, request).await; + } + + let request = table_commit_request_from_rest_request(bucket, namespace, table, request)?; + let Some(current) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &request.new_metadata_location) { + return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); + } + let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.new_metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; + validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; + let result = store.commit_table(request).await.map_err(catalog_store_error)?; + Ok(commit_table_response_from_result(result, target_metadata)) +} + +async fn standard_commit_table_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: RestCommitTableRequest, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(current) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + validate_table_commit_requirements(¤t_metadata, &request.requirements)?; + let expected_metadata = current_metadata.clone(); + let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location); + let next_metadata = apply_table_commit_updates(current_metadata, &request.updates, &previous_metadata_location)?; + validate_metadata_table_location_in_bucket(bucket, &next_metadata)?; + validate_metadata_matches_current_metadata(&expected_metadata, &next_metadata)?; + validate_table_snapshot_commit_conflicts( + metadata_backend, + bucket, + namespace, + &table_name, + ¤t, + &expected_metadata, + &request.updates, + ) + .await?; + let (commit_id, metadata_file_token) = standard_commit_ids(request.commit_id); + let next_generation = current.generation.saturating_add(1); + let next_metadata_location = crate::table_catalog::default_table_metadata_file_path( + namespace, + &table_name, + &next_metadata_file_name(next_generation, &metadata_file_token), + ); + let next_metadata_data = serde_json::to_vec(&next_metadata) + .map_err(|err| s3_error!(InternalError, "failed to serialize table metadata update: {}", err))?; + metadata_backend + .put_object( + bucket, + &next_metadata_location, + next_metadata_data, + crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, + ) + .await + .map_err(catalog_store_error)?; + + let commit_request = crate::table_catalog::TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + commit_id, + idempotency_key: request.idempotency_key, + operation: request.operation.unwrap_or_else(|| table_commit_operation(&next_metadata)), + expected_version_token: current.version_token, + expected_metadata_location: current.metadata_location, + new_metadata_location: next_metadata_location, + requirements: request.requirements, + writer: request.writer, + }; + let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; + Ok(commit_table_response_from_result(result, next_metadata)) +} + +async fn drop_table_in_store(store: &S, bucket: &str, namespace: &crate::table_catalog::Namespace, table: &str) -> S3Result<()> +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + store + .drop_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error) +} + +async fn drop_view_in_store(store: &S, bucket: &str, namespace: &crate::table_catalog::Namespace, view: &str) -> S3Result<()> +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + store + .drop_view(bucket, &namespace.public_name(), view) + .await + .map_err(catalog_store_error) +} + +async fn table_metadata_maintenance_response( + store: &crate::table_catalog::ObjectTableCatalogStore, + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: TableMetadataMaintenanceRequest, +) -> S3Result +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + if request.delete && request.commit_snapshot_expiration { + return Err(s3_error!( + InvalidRequest, + "snapshot expiration commit cannot be combined with metadata deletion" + )); + } + if request.delete && request.commit_compaction { + return Err(s3_error!(InvalidRequest, "compaction commit cannot be combined with metadata deletion")); + } + if request.commit_snapshot_expiration && request.commit_compaction { + return Err(s3_error!( + InvalidRequest, + "compaction commit cannot be combined with snapshot expiration commit" + )); + } + if request.commit_compaction && request.compaction.is_none() { + return Err(s3_error!(InvalidRequest, "commit-compaction requires a compaction request")); + } + + let snapshot_expiration_request = request.snapshot_expiration; + let commit_snapshot_expiration = request.commit_snapshot_expiration; + let compaction_request = request.compaction; + let commit_compaction = request.commit_compaction; + let compaction = match compaction_request { + Some(config) if commit_compaction => Some( + store + .commit_table_compaction(bucket, &namespace.public_name(), table, config) + .await + .map_err(catalog_store_error)?, + ), + Some(config) => Some( + store + .plan_table_compaction(bucket, &namespace.public_name(), table, config) + .await + .map_err(catalog_store_error)?, + ), + None => None, + }; + let snapshot_expiration_plan = match snapshot_expiration_request { + Some(config) => Some( + store + .plan_table_snapshot_expiration(bucket, &namespace.public_name(), table, config) + .await + .map_err(catalog_store_error)?, + ), + None => None, + }; + let mut report = store + .run_table_metadata_maintenance_with_retention( + bucket, + &namespace.public_name(), + table, + request.delete, + Some("rustfs-admin".to_string()), + request.retain_recent_metadata_files, + ) + .await + .map_err(catalog_store_error)?; + let snapshot_expiration = match (snapshot_expiration_plan, commit_snapshot_expiration) { + (Some(plan), true) => { + Some(commit_table_snapshot_expiration_response(store, metadata_backend, bucket, namespace, table, plan).await?) + } + (Some(plan), false) => Some(plan), + (None, _) => None, + }; + report.snapshot_expiration = snapshot_expiration; + report.compaction = compaction; + if report.snapshot_expiration.is_some() || report.compaction.is_some() { + let committed_snapshot_expiration = report + .snapshot_expiration + .as_ref() + .is_some_and(|snapshot_expiration| snapshot_expiration.committed_metadata_location.is_some()); + let committed_compaction = report + .compaction + .as_ref() + .is_some_and(|compaction| compaction.committed_metadata_location.is_some()); + match store.put_table_metadata_maintenance_report(&report).await { + Ok(()) => {} + Err(err) if committed_snapshot_expiration || committed_compaction => { + tracing::warn!( + error = %err, + warehouse = bucket, + namespace = namespace.public_name(), + table, + "failed to persist table maintenance report after catalog maintenance commit" + ); + } + Err(err) => return Err(catalog_store_error(err)), + } + } + Ok(report) +} + +async fn commit_table_snapshot_expiration_response( + store: &crate::table_catalog::ObjectTableCatalogStore, + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + mut report: crate::table_catalog::TableSnapshotExpirationReport, +) -> S3Result +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let Some(current) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + if report.table_id != current.table_id || report.current_metadata_location != current.metadata_location { + return Err(s3_error!(PreconditionFailed, "snapshot expiration plan is stale")); + } + if report.manual_review_count > 0 { + return Err(s3_error!(InvalidRequest, "snapshot expiration plan requires manual review before commit")); + } + let expired_snapshot_ids = report + .snapshot_reports + .iter() + .filter(|snapshot| snapshot.state == crate::table_catalog::TableSnapshotExpirationSnapshotState::ExpirationCandidate) + .filter_map(|snapshot| snapshot.snapshot_id) + .collect::>(); + if expired_snapshot_ids.is_empty() { + return Ok(report); + } + + let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + let updates = [serde_json::json!({ + "action": "remove-snapshots", + "snapshot-ids": expired_snapshot_ids.clone() + })]; + let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location); + let next_metadata = apply_table_commit_updates(current_metadata.clone(), &updates, &previous_metadata_location)?; + validate_metadata_matches_current_metadata(¤t_metadata, &next_metadata)?; + validate_metadata_table_location_in_bucket(bucket, &next_metadata)?; + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + let (commit_id, metadata_file_token) = standard_commit_ids(None); + let next_generation = current.generation.saturating_add(1); + let next_metadata_location = crate::table_catalog::default_table_metadata_file_path( + namespace, + &table_name, + &next_metadata_file_name(next_generation, &metadata_file_token), + ); + let next_metadata_data = serde_json::to_vec(&next_metadata) + .map_err(|err| s3_error!(InternalError, "failed to serialize snapshot expiration metadata: {}", err))?; + metadata_backend + .put_object( + bucket, + &next_metadata_location, + next_metadata_data, + crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, + ) + .await + .map_err(catalog_store_error)?; + + let commit_request = crate::table_catalog::TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + commit_id, + idempotency_key: None, + operation: "expire-snapshots".to_string(), + expected_version_token: current.version_token, + expected_metadata_location: current.metadata_location, + new_metadata_location: next_metadata_location, + requirements: Vec::new(), + writer: Some("rustfs-maintenance".to_string()), + }; + let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; + report.expired_snapshot_ids = expired_snapshot_ids; + report.committed_metadata_location = Some(result.table.metadata_location); + Ok(report) +} + +async fn table_refs_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(entry) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); + let refs = metadata + .get("refs") + .and_then(serde_json::Value::as_object) + .cloned() + .unwrap_or_default() + .into_iter() + .collect::>(); + let protected_ref_count = refs + .values() + .filter(|reference| { + reference + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_some_and(|snapshot_id| Some(snapshot_id) != current_snapshot_id) + }) + .count(); + let user_defined_ref_count = refs.keys().filter(|name| name.as_str() != "main").count(); + + Ok(TableRefsResponse { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + current_metadata_location: entry.metadata_location, + current_snapshot_id, + protected_ref_count, + user_defined_ref_count, + refs, + }) +} + +async fn put_table_ref_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + ref_name: &str, + request: PutTableRefRequest, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + if !matches!(request.ref_type.as_str(), "branch" | "tag") { + return Err(s3_error!(InvalidRequest, "snapshot ref type must be branch or tag")); + } + let mut update = serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": ref_name, + "type": request.ref_type, + "snapshot-id": request.snapshot_id + }); + if let Some(value) = request.min_snapshots_to_keep { + update["min-snapshots-to-keep"] = serde_json::Value::from(value); + } + if let Some(value) = request.max_snapshot_age_ms { + update["max-snapshot-age-ms"] = serde_json::Value::from(value); + } + if let Some(value) = request.max_ref_age_ms { + update["max-ref-age-ms"] = serde_json::Value::from(value); + } + let mut requirements = Vec::new(); + if let Some(expected_snapshot_id) = request.expected_snapshot_id { + requirements.push(serde_json::json!({ + "type": "assert-ref-snapshot-id", + "ref": ref_name, + "snapshot-id": expected_snapshot_id + })); + } + standard_commit_table_response( + store, + metadata_backend, + bucket, + namespace, + table, + RestCommitTableRequest { + _identifier: None, + commit_id: request.commit_id, + idempotency_key: request.idempotency_key, + operation: Some("set-snapshot-ref".to_string()), + expected_version_token: None, + expected_metadata_location: None, + new_metadata_location: None, + requirements, + updates: vec![update], + writer: request.writer.or_else(|| Some("rustfs-ref-api".to_string())), + }, + ) + .await +} + +async fn delete_table_ref_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + ref_name: &str, + request: DeleteTableRefRequest, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + if ref_name == "main" { + return Err(s3_error!(InvalidRequest, "main snapshot ref cannot be deleted")); + } + let Some(entry) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let reference = metadata + .get("refs") + .and_then(serde_json::Value::as_object) + .and_then(|refs| refs.get(ref_name)); + if reference.is_some_and(snapshot_ref_has_explicit_retention) && !request.force { + return Err(s3_error!(InvalidRequest, "snapshot ref has retention policy; force is required")); + } + let mut requirements = Vec::new(); + if let Some(expected_snapshot_id) = request.expected_snapshot_id { + requirements.push(serde_json::json!({ + "type": "assert-ref-snapshot-id", + "ref": ref_name, + "snapshot-id": expected_snapshot_id + })); + } + standard_commit_table_response( + store, + metadata_backend, + bucket, + namespace, + table, + RestCommitTableRequest { + _identifier: None, + commit_id: request.commit_id, + idempotency_key: request.idempotency_key, + operation: Some("remove-snapshot-ref".to_string()), + expected_version_token: None, + expected_metadata_location: None, + new_metadata_location: None, + requirements, + updates: vec![serde_json::json!({ + "action": "remove-snapshot-ref", + "ref-name": ref_name + })], + writer: request.writer.or_else(|| Some("rustfs-ref-api".to_string())), + }, + ) + .await +} + +fn snapshot_ref_has_explicit_retention(reference: &serde_json::Value) -> bool { + reference.get("min-snapshots-to-keep").is_some() + || reference.get("max-snapshot-age-ms").is_some() + || reference.get("max-ref-age-ms").is_some() +} + +async fn external_catalog_bridge_response( + store: &crate::table_catalog::ObjectTableCatalogStore, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, +) -> S3Result +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let bridge = store + .get_external_catalog_bridge(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)?; + Ok(external_catalog_bridge_response_from_entry(bucket, namespace, table, bridge)) +} + +fn external_catalog_bridge_response_from_entry( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + bridge: Option, +) -> ExternalCatalogBridgeResponse { + let status = if bridge.is_some() { + EXTERNAL_CATALOG_BRIDGE_STATUS_CONFIGURED + } else { + EXTERNAL_CATALOG_BRIDGE_STATUS_UNCONFIGURED + }; + ExternalCatalogBridgeResponse { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + status: status.to_string(), + supported_import: + "register/import an existing Iceberg metadata location into the RustFS catalog; operator-supplied external metadata sync is supported" + .to_string(), + capabilities: external_catalog_bridge_capabilities(), + unsupported_bridges: Vec::new(), + bridge: bridge.map(external_catalog_bridge_state_response), + } +} + +fn external_catalog_bridge_capabilities() -> Vec { + EXTERNAL_CATALOG_BRIDGE_CAPABILITIES + .iter() + .map(|catalog| ExternalCatalogBridgeCapability { + catalog: (*catalog).to_string(), + status: EXTERNAL_CATALOG_BRIDGE_SUPPORTED_STATUS.to_string(), + reason: EXTERNAL_CATALOG_BRIDGE_SUPPORTED_REASON.to_string(), + }) + .collect() +} + +fn external_catalog_bridge_state_response( + entry: crate::table_catalog::ExternalCatalogBridgeEntry, +) -> ExternalCatalogBridgeStateResponse { + ExternalCatalogBridgeStateResponse { + catalog: entry.catalog, + external_catalog_id: entry.external_catalog_id, + external_namespace: entry.external_namespace, + external_table: entry.external_table, + external_table_uuid: entry.external_table_uuid, + metadata_location: entry.metadata_location, + external_version_token: entry.external_version_token, + policy_mode: entry.policy_mode, + credential_mode: entry.credential_mode, + sync_mode: entry.sync_mode, + rollback_strategy: entry.rollback_strategy, + last_sync_status: entry.last_sync_status, + last_synced_metadata_location: entry.last_synced_metadata_location, + properties: entry.properties, + } +} + +fn validate_external_catalog_name(catalog: String) -> S3Result { + let catalog = catalog.trim().to_ascii_lowercase(); + if EXTERNAL_CATALOG_BRIDGE_CAPABILITIES.contains(&catalog.as_str()) { + return Ok(catalog); + } + Err(s3_error!(InvalidRequest, "unsupported external catalog bridge: {catalog}")) +} + +fn validate_external_catalog_field(field_name: &str, value: String) -> S3Result { + let value = value.trim().to_string(); + if value.is_empty() { + return Err(s3_error!(InvalidRequest, "{} must not be empty", field_name)); + } + if value.len() > 512 || value.chars().any(|ch| ch.is_control() || ch == '/' || ch == '\\') { + return Err(s3_error!(InvalidRequest, "{} contains unsupported characters", field_name)); + } + Ok(value) +} + +fn validate_external_catalog_optional_field(field_name: &str, value: Option) -> S3Result> { + value + .map(|value| validate_external_catalog_field(field_name, value)) + .transpose() +} + +fn validate_external_catalog_mode(field_name: &str, value: Option, default_value: &str) -> S3Result { + match value { + Some(value) if value == default_value => Ok(value), + Some(_) => Err(s3_error!( + InvalidRequest, + "{} must be {} until additional external catalog modes are implemented", + field_name, + default_value + )), + None => Ok(default_value.to_string()), + } +} + +fn validate_external_catalog_metadata_location( + namespace: &crate::table_catalog::Namespace, + table: &str, + metadata_location: &str, +) -> S3Result<()> { + let table = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table, metadata_location) { + return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); + } + Ok(()) +} + +fn validate_external_catalog_metadata_uuid( + request_table_uuid: Option<&str>, + metadata: &serde_json::Value, +) -> S3Result> { + let metadata_table_uuid = metadata_table_uuid(metadata)?; + if let Some(request_table_uuid) = request_table_uuid + && request_table_uuid != metadata_table_uuid + { + return Err(s3_error!(InvalidRequest, "external table uuid does not match table metadata table-uuid")); + } + Ok(Some(metadata_table_uuid.to_string())) +} + +fn external_catalog_bridge_entry_from_request( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: ExternalCatalogBridgeRequest, +) -> S3Result { + let catalog = validate_external_catalog_name(request.catalog)?; + let external_catalog_id = validate_external_catalog_optional_field("external catalog id", request.external_catalog_id)?; + let external_namespace = validate_external_catalog_field("external namespace", request.external_namespace)?; + let external_table = validate_external_catalog_field("external table", request.external_table)?; + let external_table_uuid = validate_external_catalog_optional_field("external table uuid", request.external_table_uuid)?; + let metadata_location = request + .metadata_location + .map(|metadata_location| table_metadata_location_for_catalog(bucket, &metadata_location)) + .transpose()?; + if let Some(metadata_location) = metadata_location.as_deref() { + validate_external_catalog_metadata_location(namespace, table, metadata_location)?; + } + Ok(crate::table_catalog::ExternalCatalogBridgeEntry { + version: crate::table_catalog::TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + catalog, + external_catalog_id, + external_namespace, + external_table, + external_table_uuid, + metadata_location, + external_version_token: validate_external_catalog_optional_field( + "external version token", + request.external_version_token, + )?, + policy_mode: validate_external_catalog_mode("policy mode", request.policy_mode, EXTERNAL_CATALOG_POLICY_MODE_RUSTFS)?, + credential_mode: validate_external_catalog_mode( + "credential mode", + request.credential_mode, + EXTERNAL_CATALOG_CREDENTIAL_MODE_RUSTFS, + )?, + sync_mode: validate_external_catalog_mode("sync mode", request.sync_mode, EXTERNAL_CATALOG_SYNC_MODE_MANUAL)?, + rollback_strategy: EXTERNAL_CATALOG_ROLLBACK_RETAIN_CURRENT.to_string(), + last_sync_status: None, + last_synced_metadata_location: None, + properties: request.properties, + created_at: None, + updated_at: None, + }) +} + +fn external_catalog_bridge_entry_from_sync_request( + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: &ExternalCatalogBridgeSyncRequest, + external_table_uuid: Option, +) -> S3Result { + let catalog = validate_external_catalog_name(request.catalog.clone())?; + let external_catalog_id = + validate_external_catalog_optional_field("external catalog id", request.external_catalog_id.clone())?; + let external_namespace = validate_external_catalog_field("external namespace", request.external_namespace.clone())?; + let external_table = validate_external_catalog_field("external table", request.external_table.clone())?; + let external_version_token = + validate_external_catalog_optional_field("external version token", request.external_version_token.clone())?; + let rollback_strategy = validate_external_catalog_mode( + "rollback strategy", + request.rollback_strategy.clone(), + EXTERNAL_CATALOG_ROLLBACK_RETAIN_CURRENT, + )?; + Ok(crate::table_catalog::ExternalCatalogBridgeEntry { + version: crate::table_catalog::TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + catalog, + external_catalog_id, + external_namespace, + external_table, + external_table_uuid, + metadata_location: Some(request.metadata_location.clone()), + external_version_token, + policy_mode: validate_external_catalog_mode( + "policy mode", + request.policy_mode.clone(), + EXTERNAL_CATALOG_POLICY_MODE_RUSTFS, + )?, + credential_mode: validate_external_catalog_mode( + "credential mode", + request.credential_mode.clone(), + EXTERNAL_CATALOG_CREDENTIAL_MODE_RUSTFS, + )?, + sync_mode: EXTERNAL_CATALOG_SYNC_MODE_MANUAL.to_string(), + rollback_strategy, + last_sync_status: Some(EXTERNAL_CATALOG_BRIDGE_SYNC_STATUS.to_string()), + last_synced_metadata_location: Some(request.metadata_location.clone()), + properties: request.properties.clone(), + created_at: None, + updated_at: None, + }) +} + +async fn put_external_catalog_bridge_response( + store: &crate::table_catalog::ObjectTableCatalogStore, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: ExternalCatalogBridgeRequest, +) -> S3Result +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let entry = external_catalog_bridge_entry_from_request(bucket, namespace, table, request)?; + let entry = store.put_external_catalog_bridge(entry).await.map_err(catalog_store_error)?; + Ok(external_catalog_bridge_response_from_entry(bucket, namespace, table, Some(entry))) +} + +async fn sync_external_catalog_bridge_response( + store: &crate::table_catalog::ObjectTableCatalogStore, + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: ExternalCatalogBridgeSyncRequest, + table_bucket_enabled: bool, +) -> S3Result +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let mut request = request; + request.metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; + request.expected_metadata_location = request + .expected_metadata_location + .map(|metadata_location| table_metadata_location_for_catalog(bucket, &metadata_location)) + .transpose()?; + ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + validate_external_catalog_metadata_location(namespace, table, &request.metadata_location)?; + let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; + let external_table_uuid = validate_external_catalog_metadata_uuid(request.external_table_uuid.as_deref(), &target_metadata)?; + + let (action, table_response) = if let Some(current) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + { + let expected_version_token = request + .expected_version_token + .clone() + .ok_or_else(|| s3_error!(InvalidRequest, "external catalog sync requires expected-version-token"))?; + let expected_metadata_location = request + .expected_metadata_location + .clone() + .ok_or_else(|| s3_error!(InvalidRequest, "external catalog sync requires expected-metadata-location"))?; + let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; + let result = store + .commit_table(crate::table_catalog::TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + commit_id: request.commit_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string()), + idempotency_key: request.idempotency_key.clone(), + operation: EXTERNAL_CATALOG_SYNC_OPERATION.to_string(), + expected_version_token, + expected_metadata_location, + new_metadata_location: request.metadata_location.clone(), + requirements: Vec::new(), + writer: Some(EXTERNAL_CATALOG_SYNC_WRITER.to_string()), + }) + .await + .map_err(catalog_store_error)?; + ( + EXTERNAL_CATALOG_ACTION_COMMITTED.to_string(), + load_table_response_from_entry(result.table, target_metadata), + ) + } else { + if request.expected_version_token.is_some() || request.expected_metadata_location.is_some() { + return Err(s3_error!( + InvalidRequest, + "external catalog sync cannot use expected table state when registering a missing table" + )); + } + let mut entry = table_entry_from_import_request( + bucket, + namespace, + table, + CatalogImportRequest { + metadata_location: request.metadata_location.clone(), + properties: request.properties.clone(), + }, + )?; + adopt_registered_metadata_identity(&mut entry, &target_metadata)?; + store.register_table(entry.clone()).await.map_err(catalog_store_error)?; + ( + EXTERNAL_CATALOG_ACTION_REGISTERED.to_string(), + load_table_response_from_entry(entry, target_metadata), + ) + }; + + let bridge_entry = external_catalog_bridge_entry_from_sync_request(bucket, namespace, table, &request, external_table_uuid)?; + let bridge_entry = store + .put_external_catalog_bridge(bridge_entry) + .await + .map_err(catalog_store_error)?; + Ok(ExternalCatalogBridgeSyncResponse { + action, + table: table_response, + bridge: external_catalog_bridge_response_from_entry(bucket, namespace, table, Some(bridge_entry)), + }) +} + +async fn catalog_import_response( + store: &S, + metadata_backend: &B, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: CatalogImportRequest, + table_bucket_enabled: bool, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, + B: crate::table_catalog::TableCatalogObjectBackend, +{ + let started = Instant::now(); + let result = async { + ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + let mut entry = table_entry_from_import_request(bucket, namespace, table, request)?; + let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &metadata)?; + adopt_registered_metadata_identity(&mut entry, &metadata)?; + if let Some(existing) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + { + if existing.table_uuid == entry.table_uuid + && existing.metadata_location == entry.metadata_location + && existing.warehouse_location == entry.warehouse_location + { + return Ok(load_table_response_from_entry(existing, metadata)); + } + return Err(s3_error!( + PreconditionFailed, + "catalog import target already exists with different table identity or metadata pointer" + )); + } + store.register_table(entry.clone()).await.map_err(catalog_store_error)?; + Ok(load_table_response_from_entry(entry, metadata)) + } + .await; + record_table_catalog_admin_operation_result("import", bucket, &namespace.public_name(), table, started, &result); + result +} + +async fn rollback_table_response( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + request: RollbackTableRequest, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let started = Instant::now(); + let result = async { + let Some(current) = store + .load_table(bucket, &namespace.public_name(), table) + .await + .map_err(catalog_store_error)? + else { + return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); + }; + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; + if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &metadata_location) { + return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); + } + let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; + validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; + let commit_request = crate::table_catalog::TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + commit_id: request.commit_id.unwrap_or_else(|| Uuid::new_v4().to_string()), + idempotency_key: request.idempotency_key, + operation: "rollback".to_string(), + expected_version_token: request.version_token, + expected_metadata_location: current.metadata_location, + new_metadata_location: metadata_location, + requirements: Vec::new(), + writer: Some("rustfs-catalog-rollback-api".to_string()), + }; + let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; + Ok(commit_table_response_from_result(result, target_metadata)) + } + .await; + record_table_catalog_admin_operation_result("rollback", bucket, &namespace.public_name(), table, started, &result); + result +} + +#[cfg(test)] +mod tests; diff --git a/rustfs/src/admin/handlers/table_catalog/namespace.rs b/rustfs/src/admin/handlers/table_catalog/namespace.rs new file mode 100644 index 000000000..9a13ec36b --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/namespace.rs @@ -0,0 +1,93 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub struct RestListNamespacesHandler {} + +#[async_trait::async_trait] +impl Operation for RestListNamespacesHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let resource = TableCatalogResource::warehouse(&warehouse); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let response = list_namespaces_response(&store, &warehouse, &req.uri).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestCreateNamespaceHandler {} + +#[async_trait::async_trait] +impl Operation for RestCreateNamespaceHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let resource = TableCatalogResource::warehouse(&warehouse); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableNamespaceAction).await?; + let request = read_json_body::(req.input).await?; + let store = table_catalog_store()?; + let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let response = create_namespace_response(&store, &warehouse, request, table_bucket_enabled).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestGetNamespaceHandler {} + +#[async_trait::async_trait] +impl Operation for RestGetNamespaceHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let resource = TableCatalogResource::namespace(&warehouse, &namespace); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let response = get_namespace_response(&store, &warehouse, &namespace).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestDropNamespaceHandler {} + +#[async_trait::async_trait] +impl Operation for RestDropNamespaceHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let resource = TableCatalogResource::namespace(&warehouse, &namespace); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableNamespaceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + drop_namespace_in_store(&store, &warehouse, &namespace.public_name()).await?; + Ok(empty_response(StatusCode::NO_CONTENT)) + } +} + +pub struct RestNamespaceExistsHandler {} + +#[async_trait::async_trait] +impl Operation for RestNamespaceExistsHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let resource = TableCatalogResource::namespace(&warehouse, &namespace); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + Ok(empty_response(namespace_exists_status(&store, &warehouse, &namespace).await?)) + } +} diff --git a/rustfs/src/admin/handlers/table_catalog/refs.rs b/rustfs/src/admin/handlers/table_catalog/refs.rs new file mode 100644 index 000000000..2d2569abd --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/refs.rs @@ -0,0 +1,75 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub struct ListTableRefsHandler {} + +#[async_trait::async_trait] +impl Operation for ListTableRefsHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = table_refs_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct PutTableRefHandler {} + +#[async_trait::async_trait] +impl Operation for PutTableRefHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let ref_name = ref_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = + put_table_ref_response(&store, &metadata_backend, &warehouse, &namespace, &table, &ref_name, request).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct DeleteTableRefHandler {} + +#[async_trait::async_trait] +impl Operation for DeleteTableRefHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let ref_name = ref_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body_or_default::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = + delete_table_ref_response(&store, &metadata_backend, &warehouse, &namespace, &table, &ref_name, request).await?; + build_json_response(StatusCode::OK, &response) + } +} diff --git a/rustfs/src/admin/handlers/table_catalog/routes.rs b/rustfs/src/admin/handlers/table_catalog/routes.rs new file mode 100644 index 000000000..f5faf157d --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/routes.rs @@ -0,0 +1,259 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub fn register_table_catalog_route(r: &mut S3Router) -> std::io::Result<()> { + for prefix in [TABLE_CATALOG_PREFIX, TABLE_CATALOG_COMPAT_PREFIX] { + register_table_catalog_prefix_routes(r, prefix)?; + } + + Ok(()) +} + +fn register_table_catalog_prefix_routes(r: &mut S3Router, prefix: &str) -> std::io::Result<()> { + r.insert(Method::GET, format!("{prefix}/config").as_str(), AdminOperation(&GET_CONFIG_HANDLER))?; + r.insert( + Method::PUT, + format!("{prefix}/buckets/{{warehouse}}").as_str(), + AdminOperation(&ENABLE_TABLE_BUCKET_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/buckets/{{warehouse}}").as_str(), + AdminOperation(&GET_TABLE_BUCKET_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/catalog/migration").as_str(), + AdminOperation(&GET_TABLE_CATALOG_MIGRATION_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/catalog/migration").as_str(), + AdminOperation(&MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER), + )?; + r.insert( + Method::DELETE, + format!("{prefix}/{{warehouse}}/catalog/migration").as_str(), + AdminOperation(&CANCEL_TABLE_CATALOG_MIGRATION_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces").as_str(), + AdminOperation(&LIST_NAMESPACES_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces").as_str(), + AdminOperation(&CREATE_NAMESPACE_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}").as_str(), + AdminOperation(&GET_NAMESPACE_HANDLER), + )?; + r.insert( + Method::HEAD, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}").as_str(), + AdminOperation(&NAMESPACE_EXISTS_HANDLER), + )?; + r.insert( + Method::DELETE, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}").as_str(), + AdminOperation(&DROP_NAMESPACE_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables").as_str(), + AdminOperation(&LIST_TABLES_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables").as_str(), + AdminOperation(&CREATE_TABLE_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/register").as_str(), + AdminOperation(®ISTER_TABLE_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views").as_str(), + AdminOperation(&LIST_VIEWS_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views").as_str(), + AdminOperation(&CREATE_VIEW_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}").as_str(), + AdminOperation(&LOAD_TABLE_HANDLER), + )?; + r.insert( + Method::HEAD, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}").as_str(), + AdminOperation(&TABLE_EXISTS_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/credentials").as_str(), + AdminOperation(&LOAD_CREDENTIALS_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}").as_str(), + AdminOperation(&COMMIT_TABLE_HANDLER), + )?; + r.insert( + Method::DELETE, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}").as_str(), + AdminOperation(&DROP_TABLE_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views/{{view}}").as_str(), + AdminOperation(&LOAD_VIEW_HANDLER), + )?; + r.insert( + Method::HEAD, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views/{{view}}").as_str(), + AdminOperation(&VIEW_EXISTS_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views/{{view}}").as_str(), + AdminOperation(&REPLACE_VIEW_HANDLER), + )?; + r.insert( + Method::DELETE, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/views/{{view}}").as_str(), + AdminOperation(&DROP_VIEW_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/refs").as_str(), + AdminOperation(&LIST_TABLE_REFS_HANDLER), + )?; + r.insert( + Method::PUT, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/refs/{{ref}}").as_str(), + AdminOperation(&PUT_TABLE_REF_HANDLER), + )?; + r.insert( + Method::DELETE, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/refs/{{ref}}").as_str(), + AdminOperation(&DELETE_TABLE_REF_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/metadata-location").as_str(), + AdminOperation(&GET_TABLE_METADATA_LOCATION_HANDLER), + )?; + r.insert( + Method::PUT, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/metadata-location").as_str(), + AdminOperation(&UPDATE_TABLE_METADATA_LOCATION_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/metadata").as_str(), + AdminOperation(&TABLE_METADATA_MAINTENANCE_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/config").as_str(), + AdminOperation(&GET_TABLE_MAINTENANCE_CONFIG_HANDLER), + )?; + r.insert( + Method::PUT, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/config").as_str(), + AdminOperation(&PUT_TABLE_MAINTENANCE_CONFIG_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/jobs/{{job}}").as_str(), + AdminOperation(&GET_TABLE_MAINTENANCE_JOB_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/scheduler").as_str(), + AdminOperation(&GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/scheduler/run").as_str(), + AdminOperation(&RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/worker/run").as_str(), + AdminOperation(&RUN_TABLE_MAINTENANCE_WORKER_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/jobs/{{job}}/heartbeat").as_str(), + AdminOperation(&HEARTBEAT_TABLE_MAINTENANCE_JOB_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/jobs/{{job}}/quarantine").as_str(), + AdminOperation(&TABLE_MAINTENANCE_QUARANTINE_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/export").as_str(), + AdminOperation(&EXPORT_TABLE_CATALOG_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/import").as_str(), + AdminOperation(&IMPORT_TABLE_CATALOG_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/external").as_str(), + AdminOperation(&EXTERNAL_CATALOG_BRIDGE_HANDLER), + )?; + r.insert( + Method::PUT, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/external").as_str(), + AdminOperation(&PUT_EXTERNAL_CATALOG_BRIDGE_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/external/sync").as_str(), + AdminOperation(&SYNC_EXTERNAL_CATALOG_BRIDGE_HANDLER), + )?; + r.insert( + Method::GET, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/diagnostics").as_str(), + AdminOperation(&GET_TABLE_CATALOG_DIAGNOSTICS_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/recovery").as_str(), + AdminOperation(&RECOVER_TABLE_CATALOG_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/catalog/rollback").as_str(), + AdminOperation(&ROLLBACK_TABLE_CATALOG_HANDLER), + )?; + + Ok(()) +} diff --git a/rustfs/src/admin/handlers/table_catalog/table.rs b/rustfs/src/admin/handlers/table_catalog/table.rs new file mode 100644 index 000000000..10471fd94 --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/table.rs @@ -0,0 +1,296 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub struct RestListTablesHandler {} + +#[async_trait::async_trait] +impl Operation for RestListTablesHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let resource = TableCatalogResource::namespace(&warehouse, &namespace); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let response = list_tables_response(&store, &warehouse, &namespace, &req.uri).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestCreateTableHandler {} + +#[async_trait::async_trait] +impl Operation for RestCreateTableHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let resource = TableCatalogResource::namespace(&warehouse, &namespace); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::CreateTableAction).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let response = + create_table_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestRegisterTableHandler {} + +#[async_trait::async_trait] +impl Operation for RestRegisterTableHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let resource = TableCatalogResource::namespace(&warehouse, &namespace); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let response = + register_table_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestLoadTableHandler {} + +#[async_trait::async_trait] +impl Operation for RestLoadTableHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = load_table_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestTableExistsHandler {} + +#[async_trait::async_trait] +impl Operation for RestTableExistsHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + Ok(empty_response(table_exists_status(&store, &warehouse, &namespace, &table).await?)) + } +} + +pub struct RestCommitTableHandler {} + +#[async_trait::async_trait] +impl Operation for RestCommitTableHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = commit_table_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestDropTableHandler {} + +#[async_trait::async_trait] +impl Operation for RestDropTableHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + drop_table_in_store(&store, &warehouse, &namespace, &table).await?; + Ok(empty_response(StatusCode::NO_CONTENT)) + } +} + +pub struct GetTableMetadataLocationHandler {} + +#[async_trait::async_trait] +impl Operation for GetTableMetadataLocationHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataLocationAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let response = get_table_metadata_location_response(&store, &warehouse, &namespace, &table).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct UpdateTableMetadataLocationHandler {} + +#[async_trait::async_trait] +impl Operation for UpdateTableMetadataLocationHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = + update_table_metadata_location_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct ExportTableCatalogHandler {} + +#[async_trait::async_trait] +impl Operation for ExportTableCatalogHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let started = Instant::now(); + let result = store + .export_table_catalog_entry(&warehouse, &namespace.public_name(), &table) + .await + .map_err(catalog_store_error); + record_table_catalog_admin_operation_result("export", &warehouse, &namespace.public_name(), &table, started, &result); + let response = result?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct ImportTableCatalogHandler {} + +#[async_trait::async_trait] +impl Operation for ImportTableCatalogHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let response = + catalog_import_response(&store, &metadata_backend, &warehouse, &namespace, &table, request, table_bucket_enabled) + .await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct GetTableCatalogDiagnosticsHandler {} + +#[async_trait::async_trait] +impl Operation for GetTableCatalogDiagnosticsHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let config = store + .get_table_maintenance_config(&warehouse, &namespace.public_name(), &table) + .await + .map_err(catalog_store_error)?; + let started = Instant::now(); + let result = store + .diagnose_table_catalog(&warehouse, &namespace.public_name(), &table, config.retain_recent_metadata_files) + .await + .map_err(catalog_store_error); + record_table_catalog_admin_operation_result( + "diagnostics", + &warehouse, + &namespace.public_name(), + &table, + started, + &result, + ); + let response = result?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RecoverTableCatalogHandler {} + +#[async_trait::async_trait] +impl Operation for RecoverTableCatalogHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let started = Instant::now(); + let result = store + .recover_table_commits(&warehouse, &namespace.public_name(), &table) + .await + .map_err(catalog_store_error); + record_table_catalog_admin_operation_result("recovery", &warehouse, &namespace.public_name(), &table, started, &result); + let response = result?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RollbackTableCatalogHandler {} + +#[async_trait::async_trait] +impl Operation for RollbackTableCatalogHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let table = table_name_from_params(¶ms)?; + let resource = TableCatalogResource::table(&warehouse, &namespace, &table); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = rollback_table_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; + build_json_response(StatusCode::OK, &response) + } +} diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs new file mode 100644 index 000000000..b83ae0e5d --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -0,0 +1,6712 @@ +use super::*; +use crate::table_catalog::{TableCatalogObjectBackend, TableCatalogStore}; +use std::sync::Arc; + +#[test] +#[serial_test::serial] +fn catalog_config_response_lists_standard_rest_endpoints() { + let response = temp_env::with_var_unset(crate::table_catalog::ENV_TABLE_CATALOG_BACKING, || catalog_config_response(None)) + .expect("catalog config should build"); + + assert_eq!(response.defaults.get(WAREHOUSE_PROPERTY).map(String::as_str), Some(DEFAULT_WAREHOUSE_ID)); + assert_eq!( + response.defaults.get(CATALOG_ENDPOINT_PREFIX_CONFIG_KEY).map(String::as_str), + Some(TABLE_CATALOG_PREFIX) + ); + assert_eq!( + response + .defaults + .get(CATALOG_COMPAT_ENDPOINT_PREFIX_CONFIG_KEY) + .map(String::as_str), + Some(TABLE_CATALOG_COMPAT_PREFIX) + ); + assert_eq!( + response.defaults.get(CATALOG_BACKING_CONFIG_KEY).map(String::as_str), + Some(crate::table_catalog::TABLE_CATALOG_BACKING_OBJECT) + ); + assert!(!response.defaults.contains_key(PREFIX_PROPERTY)); + assert!(response.overrides.is_empty()); + assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities"); + assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot"); + assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog"); + assert!(response.endpoints.contains(&"GET /v1/{prefix}/namespaces")); + assert!(response.endpoints.contains(&"GET /{warehouse}/catalog/migration")); + assert!(response.endpoints.contains(&"POST /{warehouse}/catalog/migration")); + assert!(response.endpoints.contains(&"DELETE /{warehouse}/catalog/migration")); + assert!(response.endpoints.contains(&"HEAD /v1/{prefix}/namespaces/{namespace}")); + assert!( + response + .endpoints + .contains(&"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}") + ); + assert!( + response + .endpoints + .contains(&"HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}") + ); + assert!( + response + .endpoints + .contains(&"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials") + ); + assert!(response.endpoints.contains(&"GET /{warehouse}/namespaces")); + assert!(response.endpoints.contains(&"POST /{warehouse}/namespaces")); + assert!(response.endpoints.contains(&"HEAD /{warehouse}/namespaces/{namespace}")); + assert!( + response + .endpoints + .contains(&"POST /{warehouse}/namespaces/{namespace}/register") + ); + assert!( + response + .endpoints + .contains(&"POST /{warehouse}/namespaces/{namespace}/tables") + ); + assert!(response.endpoints.contains(&"GET /{warehouse}/namespaces/{namespace}/views")); + assert!(response.endpoints.contains(&"POST /{warehouse}/namespaces/{namespace}/views")); + assert!( + response + .endpoints + .contains(&"HEAD /{warehouse}/namespaces/{namespace}/views/{view}") + ); + assert!( + response + .endpoints + .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}") + ); + assert!( + response + .endpoints + .contains(&"HEAD /{warehouse}/namespaces/{namespace}/tables/{table}") + ); + assert!( + response + .endpoints + .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}") + ); + assert!( + response + .endpoints + .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/credentials") + ); + assert!( + response + .endpoints + .contains(&"GET /{warehouse}/namespaces/{namespace}/views/{view}") + ); + assert!( + response + .endpoints + .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/refs") + ); + assert!( + response + .endpoints + .contains(&"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}") + ); + assert!( + response + .endpoints + .contains(&"DELETE /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}") + ); + assert!( + response + .endpoints + .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external") + ); + assert!( + response + .endpoints + .contains(&"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external") + ); + assert!( + response + .endpoints + .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external/sync") + ); + assert!( + response + .endpoints + .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/recovery") + ); +} + +#[test] +#[serial_test::serial] +fn catalog_config_response_reports_durable_strong_backing_override() { + let response = temp_env::with_var( + crate::table_catalog::ENV_TABLE_CATALOG_BACKING, + Some(crate::table_catalog::TABLE_CATALOG_BACKING_DURABLE_STRONG), + || catalog_config_response(None), + ) + .expect("catalog config should build"); + + assert_eq!( + response.overrides.get(CATALOG_BACKING_CONFIG_KEY).map(String::as_str), + Some(crate::table_catalog::TABLE_CATALOG_BACKING_DURABLE_STRONG) + ); +} + +#[test] +#[serial_test::serial] +fn catalog_config_response_uses_requested_warehouse_as_standard_prefix() { + let response = temp_env::with_var_unset(crate::table_catalog::ENV_TABLE_CATALOG_BACKING, || { + catalog_config_response(Some("analytics")) + }) + .expect("catalog config should build"); + + assert_eq!(response.defaults.get(PREFIX_PROPERTY).map(String::as_str), Some("analytics")); +} + +#[test] +fn warehouse_config_query_rejects_empty_and_repeated_values() { + let uri = "/iceberg/v1/config?warehouse=analytics".parse().expect("URI"); + assert_eq!(warehouse_from_config_query(&uri).expect("warehouse query"), Some("analytics".to_string())); + + let uri = "/iceberg/v1/config?warehouse=".parse().expect("URI"); + assert!(warehouse_from_config_query(&uri).is_err()); + + let uri = "/iceberg/v1/config?warehouse=one&warehouse=two".parse().expect("URI"); + assert!(warehouse_from_config_query(&uri).is_err()); +} + +#[test] +fn catalog_conflicts_use_operation_specific_iceberg_errors() { + let already_exists = catalog_store_already_exists_error(crate::table_catalog::TableCatalogStoreError::Conflict( + "table already exists".to_string(), + )); + assert_eq!(already_exists.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_ALREADY_EXISTS.into())); + assert_eq!(already_exists.status_code(), Some(StatusCode::CONFLICT)); + + let namespace_not_empty = catalog_store_namespace_drop_error(crate::table_catalog::TableCatalogStoreError::Conflict( + "namespace is not empty".to_string(), + )); + assert_eq!(namespace_not_empty.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_NAMESPACE_NOT_EMPTY.into())); + assert_eq!(namespace_not_empty.status_code(), Some(StatusCode::CONFLICT)); + + let namespace_not_found = catalog_store_namespace_drop_error(crate::table_catalog::TableCatalogStoreError::NotFound( + "namespace not found".to_string(), + )); + assert_eq!(namespace_not_found.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_NO_SUCH_NAMESPACE.into())); + assert_eq!(namespace_not_found.status_code(), Some(StatusCode::NOT_FOUND)); +} + +#[test] +fn table_catalog_admin_operation_result_labels_are_stable() { + let success: Result<(), ()> = Ok(()); + let failure: Result<(), ()> = Err(()); + + assert_eq!(table_catalog_admin_operation_result_label(&success), "success"); + assert_eq!(table_catalog_admin_operation_result_label(&failure), "failure"); +} + +#[test] +fn table_catalog_handlers_require_table_admin_actions() { + let src = table_catalog_handler_source(); + + assert!( + operation_block(&src, "GetCatalogConfigHandler") + .contains("authorize_table_catalog_request(&req, AdminAction::GetTableCatalogAction).await?;") + ); + assert!( + src.contains("validate_admin_request_with_bucket_object("), + "catalog resource auth should pass namespace/table scope into IAM object matching" + ); + + for (handler, action) in [ + ("EnableTableBucketHandler", "AdminAction::SetTableBucketAction"), + ("GetTableBucketHandler", "AdminAction::GetTableBucketAction"), + ("GetTableCatalogMigrationHandler", "AdminAction::GetTableCatalogAction"), + ("RestListNamespacesHandler", "AdminAction::GetTableNamespaceAction"), + ("RestCreateNamespaceHandler", "AdminAction::SetTableNamespaceAction"), + ("RestGetNamespaceHandler", "AdminAction::GetTableNamespaceAction"), + ("RestNamespaceExistsHandler", "AdminAction::GetTableNamespaceAction"), + ("RestDropNamespaceHandler", "AdminAction::DeleteTableNamespaceAction"), + ("RestListTablesHandler", "AdminAction::GetTableAction"), + ("RestCreateTableHandler", "AdminAction::CreateTableAction"), + ("RestRegisterTableHandler", "AdminAction::RegisterTableAction"), + ("RestListViewsHandler", "AdminAction::GetTableMetadataAction"), + ("RestCreateViewHandler", "AdminAction::CreateTableAction"), + ("RestLoadTableHandler", "AdminAction::GetTableMetadataAction"), + ("RestTableExistsHandler", "AdminAction::GetTableAction"), + ("RestLoadCredentialsHandler", "AdminAction::GetTableCredentialsAction"), + ("RestCommitTableHandler", "AdminAction::CommitTableAction"), + ("RestDropTableHandler", "AdminAction::DeleteTableAction"), + ("RestLoadViewHandler", "AdminAction::GetTableMetadataAction"), + ("RestReplaceViewHandler", "AdminAction::CommitTableAction"), + ("RestDropViewHandler", "AdminAction::DeleteTableAction"), + ("ListTableRefsHandler", "AdminAction::GetTableMetadataAction"), + ("GetTableMetadataLocationHandler", "AdminAction::GetTableMetadataLocationAction"), + ("UpdateTableMetadataLocationHandler", "AdminAction::SetTableMetadataLocationAction"), + ("RestTableMetadataMaintenanceHandler", "AdminAction::RunTableMaintenanceAction"), + ("GetTableMaintenanceConfigHandler", "AdminAction::GetTableLifecycleAction"), + ("PutTableMaintenanceConfigHandler", "AdminAction::SetTableLifecycleAction"), + ("GetTableMaintenanceJobHandler", "AdminAction::GetTableLifecycleAction"), + ("GetTableMaintenanceSchedulerHandler", "AdminAction::GetTableLifecycleAction"), + ("RunTableMaintenanceSchedulerHandler", "AdminAction::RunTableMaintenanceAction"), + ("TableMaintenanceQuarantineHandler", "AdminAction::RunTableMaintenanceAction"), + ("ExportTableCatalogHandler", "AdminAction::GetTableMetadataAction"), + ("ImportTableCatalogHandler", "AdminAction::RegisterTableAction"), + ("ExternalCatalogBridgeHandler", "AdminAction::GetTableMetadataAction"), + ("PutExternalCatalogBridgeHandler", "AdminAction::RegisterTableAction"), + ("SyncExternalCatalogBridgeHandler", "AdminAction::SetTableMetadataLocationAction"), + ("GetTableCatalogDiagnosticsHandler", "AdminAction::GetTableMetadataAction"), + ("RecoverTableCatalogHandler", "AdminAction::CommitTableAction"), + ("RollbackTableCatalogHandler", "AdminAction::CommitTableAction"), + ] { + let block = operation_block(&src, handler); + assert!( + block.contains(&format!("authorize_table_catalog_resource_request(&req, &resource, {action}).await?;")), + "{handler} should require {action} with catalog resource auth" + ); + assert!( + !block.contains("authorize_table_catalog_request(&req,"), + "{handler} must not use unscoped table catalog authorization" + ); + assert!( + !block.contains("authorize_table_catalog_warehouse_request(&req, &warehouse,"), + "{handler} should not bypass catalog resource auth" + ); + } + + let sync_bridge_block = operation_block(&src, "SyncExternalCatalogBridgeHandler"); + assert!( + sync_bridge_block.contains("AdminAction::RegisterTableAction"), + "external catalog sync should require register authorization before creating a missing table" + ); + assert!( + sync_bridge_block.contains(".load_table(&warehouse, &namespace.public_name(), &table)"), + "external catalog sync should branch authorization on current table existence" + ); + + let migration_block = operation_block(&src, "GetTableCatalogMigrationHandler"); + assert!( + migration_block.contains("TableCatalogResource::warehouse(&warehouse)"), + "catalog migration dry-run should authorize against the warehouse resource" + ); + for handler in [ + "MaterializeTableCatalogMigrationHandler", + "CancelTableCatalogMigrationHandler", + ] { + let block = operation_block(&src, handler); + assert!( + block.contains("authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;"), + "{handler} should require the global catalog migration action" + ); + assert!( + !block.contains("authorize_table_catalog_resource_request("), + "{handler} must not imply that a global backing cutover is warehouse-scoped" + ); + } + + for (handler, action) in [ + ("RestLoadTableHandler", "AdminAction::GetTableMetadataAction"), + ("RestTableExistsHandler", "AdminAction::GetTableAction"), + ("RestLoadCredentialsHandler", "AdminAction::GetTableCredentialsAction"), + ("RestCommitTableHandler", "AdminAction::CommitTableAction"), + ("RestDropTableHandler", "AdminAction::DeleteTableAction"), + ("ListTableRefsHandler", "AdminAction::GetTableMetadataAction"), + ("GetTableMetadataLocationHandler", "AdminAction::GetTableMetadataLocationAction"), + ("UpdateTableMetadataLocationHandler", "AdminAction::SetTableMetadataLocationAction"), + ("RestTableMetadataMaintenanceHandler", "AdminAction::RunTableMaintenanceAction"), + ("GetTableMaintenanceConfigHandler", "AdminAction::GetTableLifecycleAction"), + ("PutTableMaintenanceConfigHandler", "AdminAction::SetTableLifecycleAction"), + ("GetTableMaintenanceJobHandler", "AdminAction::GetTableLifecycleAction"), + ("GetTableMaintenanceSchedulerHandler", "AdminAction::GetTableLifecycleAction"), + ("RunTableMaintenanceSchedulerHandler", "AdminAction::RunTableMaintenanceAction"), + ("TableMaintenanceQuarantineHandler", "AdminAction::RunTableMaintenanceAction"), + ("ExportTableCatalogHandler", "AdminAction::GetTableMetadataAction"), + ("ImportTableCatalogHandler", "AdminAction::RegisterTableAction"), + ("ExternalCatalogBridgeHandler", "AdminAction::GetTableMetadataAction"), + ("PutExternalCatalogBridgeHandler", "AdminAction::RegisterTableAction"), + ("SyncExternalCatalogBridgeHandler", "AdminAction::SetTableMetadataLocationAction"), + ("GetTableCatalogDiagnosticsHandler", "AdminAction::GetTableMetadataAction"), + ("RecoverTableCatalogHandler", "AdminAction::CommitTableAction"), + ("RollbackTableCatalogHandler", "AdminAction::CommitTableAction"), + ] { + let block = operation_block(&src, handler); + assert!( + block.contains("TableCatalogResource::table(&warehouse, &namespace, &table)"), + "{handler} should build a table-aware catalog resource" + ); + assert!( + block.contains(&format!("authorize_table_catalog_resource_request(&req, &resource, {action}).await?;")), + "{handler} should authorize against the table-aware catalog resource" + ); + } +} + +#[test] +fn table_catalog_list_handlers_parse_standard_pagination() { + let src = table_catalog_handler_source(); + for (handler, helper_call) in [ + ( + "RestListNamespacesHandler", + "list_namespaces_response(&store, &warehouse, &req.uri).await?", + ), + ( + "RestListTablesHandler", + "list_tables_response(&store, &warehouse, &namespace, &req.uri).await?", + ), + ( + "RestListViewsHandler", + "list_views_response(&store, &warehouse, &namespace, &req.uri).await?", + ), + ] { + let block = operation_block(&src, handler); + assert!( + block.contains(helper_call), + "{handler} should pass the request URI to its paginated list helper" + ); + } +} + +#[test] +fn table_catalog_handlers_require_enabled_table_bucket_marker_before_catalog_state() { + let src = table_catalog_handler_source(); + + for handler in [ + "GetTableCatalogMigrationHandler", + "MaterializeTableCatalogMigrationHandler", + "CancelTableCatalogMigrationHandler", + "RestListNamespacesHandler", + "RestCreateNamespaceHandler", + "RestGetNamespaceHandler", + "RestNamespaceExistsHandler", + "RestDropNamespaceHandler", + "RestListTablesHandler", + "RestCreateTableHandler", + "RestRegisterTableHandler", + "RestListViewsHandler", + "RestCreateViewHandler", + "RestLoadTableHandler", + "RestTableExistsHandler", + "RestLoadCredentialsHandler", + "RestCommitTableHandler", + "RestDropTableHandler", + "RestLoadViewHandler", + "RestViewExistsHandler", + "RestReplaceViewHandler", + "RestDropViewHandler", + "ListTableRefsHandler", + "PutTableRefHandler", + "DeleteTableRefHandler", + "GetTableMetadataLocationHandler", + "UpdateTableMetadataLocationHandler", + "RestTableMetadataMaintenanceHandler", + "GetTableMaintenanceConfigHandler", + "PutTableMaintenanceConfigHandler", + "GetTableMaintenanceJobHandler", + "GetTableMaintenanceSchedulerHandler", + "RunTableMaintenanceSchedulerHandler", + "RunTableMaintenanceWorkerHandler", + "HeartbeatTableMaintenanceJobHandler", + "TableMaintenanceQuarantineHandler", + "ExportTableCatalogHandler", + "ImportTableCatalogHandler", + "ExternalCatalogBridgeHandler", + "PutExternalCatalogBridgeHandler", + "SyncExternalCatalogBridgeHandler", + "GetTableCatalogDiagnosticsHandler", + "RecoverTableCatalogHandler", + "RollbackTableCatalogHandler", + ] { + let block = operation_block(&src, handler); + assert!( + block.contains("ensure_table_bucket_enabled(&warehouse).await?;") + || block.contains("table_bucket_enabled_from_metadata(&warehouse).await?;"), + "{handler} should require the table bucket metadata marker before catalog state access" + ); + } +} + +#[test] +fn enable_table_bucket_response_writes_metadata_marker_before_catalog_entry() { + let src = table_catalog_handler_source(); + let block = function_block(&src, "async fn enable_table_bucket_response"); + let marker_write = block + .find("enable_table_bucket_marker(bucket).await?;") + .expect("enable should write the metadata marker"); + let catalog_entry_write = block + .find("ensure_table_bucket_entry(store, bucket, true).await?;") + .expect("enable should write the catalog entry"); + + assert!( + marker_write < catalog_entry_write, + "enable should write the metadata marker before the catalog entry" + ); +} + +#[test] +fn table_catalog_resource_builds_policy_object_scope() { + let namespace = crate::table_catalog::Namespace::parse("analytics.daily_events").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + + assert_eq!(TableCatalogResource::warehouse("warehouse-a").object_path(), None); + assert_eq!( + TableCatalogResource::namespace("warehouse-a", &namespace) + .object_path() + .as_deref(), + Some("namespaces/analytics/daily_events") + ); + assert_eq!( + TableCatalogResource::table("warehouse-a", &namespace, table.as_str()) + .object_path() + .as_deref(), + Some("namespaces/analytics/daily_events/tables/events") + ); +} + +fn operation_block<'a>(src: &'a str, handler: &str) -> &'a str { + let marker = format!("impl Operation for {handler}"); + let block = src.split_once(&marker).expect("handler impl should exist").1; + let end = block + .find("\npub struct ") + .or_else(|| block.find("\n#[cfg(test)]")) + .unwrap_or(block.len()); + &block[..end] +} + +fn table_catalog_handler_source() -> String { + [ + include_str!("mod.rs"), + include_str!("config.rs"), + include_str!("credentials.rs"), + include_str!("maintenance.rs"), + include_str!("namespace.rs"), + include_str!("refs.rs"), + include_str!("routes.rs"), + include_str!("table.rs"), + include_str!("view.rs"), + ] + .join("\n") +} + +fn function_block<'a>(src: &'a str, signature: &str) -> &'a str { + let block = src.split_once(signature).expect("function should exist").1; + let end = block + .find("\nfn ") + .or_else(|| block.find("\nasync fn ")) + .unwrap_or(block.len()); + &block[..end] +} + +#[test] +fn rest_catalog_mvp_routes_use_implemented_handlers() { + fn assert_operation() {} + + let _: &EnableTableBucketHandler = &ENABLE_TABLE_BUCKET_HANDLER; + let _: &GetTableBucketHandler = &GET_TABLE_BUCKET_HANDLER; + let _: &GetTableCatalogMigrationHandler = &GET_TABLE_CATALOG_MIGRATION_HANDLER; + let _: &MaterializeTableCatalogMigrationHandler = &MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER; + let _: &CancelTableCatalogMigrationHandler = &CANCEL_TABLE_CATALOG_MIGRATION_HANDLER; + let _: &RestListNamespacesHandler = &LIST_NAMESPACES_HANDLER; + let _: &RestCreateNamespaceHandler = &CREATE_NAMESPACE_HANDLER; + let _: &RestGetNamespaceHandler = &GET_NAMESPACE_HANDLER; + let _: &RestNamespaceExistsHandler = &NAMESPACE_EXISTS_HANDLER; + let _: &RestDropNamespaceHandler = &DROP_NAMESPACE_HANDLER; + let _: &RestListTablesHandler = &LIST_TABLES_HANDLER; + let _: &RestCreateTableHandler = &CREATE_TABLE_HANDLER; + let _: &RestRegisterTableHandler = ®ISTER_TABLE_HANDLER; + let _: &RestListViewsHandler = &LIST_VIEWS_HANDLER; + let _: &RestCreateViewHandler = &CREATE_VIEW_HANDLER; + let _: &RestLoadTableHandler = &LOAD_TABLE_HANDLER; + let _: &RestTableExistsHandler = &TABLE_EXISTS_HANDLER; + let _: &RestLoadCredentialsHandler = &LOAD_CREDENTIALS_HANDLER; + let _: &RestCommitTableHandler = &COMMIT_TABLE_HANDLER; + let _: &RestDropTableHandler = &DROP_TABLE_HANDLER; + let _: &RestLoadViewHandler = &LOAD_VIEW_HANDLER; + let _: &RestReplaceViewHandler = &REPLACE_VIEW_HANDLER; + let _: &RestDropViewHandler = &DROP_VIEW_HANDLER; + let _: &ListTableRefsHandler = &LIST_TABLE_REFS_HANDLER; + let _: &GetTableMetadataLocationHandler = &GET_TABLE_METADATA_LOCATION_HANDLER; + let _: &UpdateTableMetadataLocationHandler = &UPDATE_TABLE_METADATA_LOCATION_HANDLER; + let _: &RestTableMetadataMaintenanceHandler = &TABLE_METADATA_MAINTENANCE_HANDLER; + let _: &GetTableMaintenanceConfigHandler = &GET_TABLE_MAINTENANCE_CONFIG_HANDLER; + let _: &PutTableMaintenanceConfigHandler = &PUT_TABLE_MAINTENANCE_CONFIG_HANDLER; + let _: &GetTableMaintenanceJobHandler = &GET_TABLE_MAINTENANCE_JOB_HANDLER; + let _: &GetTableMaintenanceSchedulerHandler = &GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER; + let _: &RunTableMaintenanceSchedulerHandler = &RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER; + let _: &TableMaintenanceQuarantineHandler = &TABLE_MAINTENANCE_QUARANTINE_HANDLER; + let _: &ExportTableCatalogHandler = &EXPORT_TABLE_CATALOG_HANDLER; + let _: &ImportTableCatalogHandler = &IMPORT_TABLE_CATALOG_HANDLER; + let _: &ExternalCatalogBridgeHandler = &EXTERNAL_CATALOG_BRIDGE_HANDLER; + let _: &PutExternalCatalogBridgeHandler = &PUT_EXTERNAL_CATALOG_BRIDGE_HANDLER; + let _: &SyncExternalCatalogBridgeHandler = &SYNC_EXTERNAL_CATALOG_BRIDGE_HANDLER; + let _: &GetTableCatalogDiagnosticsHandler = &GET_TABLE_CATALOG_DIAGNOSTICS_HANDLER; + let _: &RecoverTableCatalogHandler = &RECOVER_TABLE_CATALOG_HANDLER; + let _: &RollbackTableCatalogHandler = &ROLLBACK_TABLE_CATALOG_HANDLER; + + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); + assert_operation::(); +} + +#[test] +fn table_metadata_maintenance_request_uses_conservative_defaults() { + let request: TableMetadataMaintenanceRequest = + serde_json::from_value(serde_json::json!({})).expect("default maintenance request should parse"); + + assert_eq!(request.retain_recent_metadata_files, 0); + assert!(!request.delete); + assert!(request.snapshot_expiration.is_none()); + assert!(!request.commit_snapshot_expiration); + assert!(request.compaction.is_none()); + assert!(!request.commit_compaction); +} + +#[test] +fn table_metadata_maintenance_request_accepts_delete_mode() { + let request: TableMetadataMaintenanceRequest = serde_json::from_value(serde_json::json!({ + "retain-recent-metadata-files": 2, + "delete": true + })) + .expect("metadata maintenance request should parse"); + + assert_eq!(request.retain_recent_metadata_files, 2); + assert!(request.delete); + assert!(request.snapshot_expiration.is_none()); + assert!(!request.commit_snapshot_expiration); + assert!(request.compaction.is_none()); + assert!(!request.commit_compaction); +} + +#[test] +fn table_metadata_maintenance_request_accepts_snapshot_and_compaction_plans() { + let request: TableMetadataMaintenanceRequest = serde_json::from_value(serde_json::json!({ + "commit-snapshot-expiration": true, + "snapshot-expiration": { + "min-snapshots-to-keep": 2, + "max-snapshot-age-ms": 3600000 + }, + "commit-compaction": true, + "compaction": { + "target-file-size-bytes": 536870912, + "small-file-threshold-bytes": 67108864, + "min-input-files": 5, + "max-rewrite-bytes-per-job": 10737418240u64 + } + })) + .expect("metadata maintenance request should parse maintenance planning config"); + + let snapshot_expiration = request + .snapshot_expiration + .expect("snapshot expiration config should be present"); + assert_eq!(snapshot_expiration.min_snapshots_to_keep, 2); + assert_eq!(snapshot_expiration.max_snapshot_age_ms, 3_600_000); + assert!(request.commit_snapshot_expiration); + assert!(request.commit_compaction); + let compaction = request.compaction.expect("compaction config should be present"); + assert_eq!(compaction.target_file_size_bytes, 536_870_912); + assert_eq!(compaction.small_file_threshold_bytes, 67_108_864); + assert_eq!(compaction.min_input_files, 5); + assert_eq!(compaction.max_rewrite_bytes_per_job, 10_737_418_240); +} + +#[test] +fn table_maintenance_scheduler_run_request_uses_stable_default_scheduler_id() { + let request: TableMaintenanceSchedulerRunRequest = + serde_json::from_value(serde_json::json!({})).expect("scheduler run request should parse"); + + assert_eq!(request.scheduler_id(), "rustfs-maintenance-scheduler"); +} + +#[tokio::test] +async fn table_maintenance_scheduler_run_request_empty_body_uses_default_scheduler_id() { + let request: TableMaintenanceSchedulerRunRequest = read_json_body_or_default(Body::empty()) + .await + .expect("bodyless scheduler run should use the default scheduler id"); + + assert_eq!(request.scheduler_id(), "rustfs-maintenance-scheduler"); +} + +#[test] +fn table_maintenance_scheduler_run_request_accepts_scheduler_id() { + let request: TableMaintenanceSchedulerRunRequest = serde_json::from_value(serde_json::json!({ + "scheduler-id": "scheduler-a" + })) + .expect("scheduler run request should parse scheduler id"); + + assert_eq!(request.scheduler_id(), "scheduler-a"); +} + +#[test] +fn table_maintenance_worker_run_request_uses_stable_default_worker_id() { + let request: TableMaintenanceWorkerRunRequest = + serde_json::from_value(serde_json::json!({})).expect("worker run request should parse"); + + assert_eq!(request.worker_id(), "rustfs-maintenance-worker"); +} + +#[test] +fn table_maintenance_worker_run_request_accepts_worker_id() { + let request: TableMaintenanceWorkerRunRequest = serde_json::from_value(serde_json::json!({ + "worker-id": "worker-a" + })) + .expect("worker run request should parse worker id"); + + assert_eq!(request.worker_id(), "worker-a"); +} + +#[test] +fn table_maintenance_heartbeat_request_requires_lease_id() { + let err = serde_json::from_value::(serde_json::json!({ + "worker-id": "worker-a" + })) + .expect_err("heartbeat request should require lease id"); + + assert!(err.to_string().contains("lease-id")); +} + +#[tokio::test] +async fn table_bucket_response_reports_catalog_discovery_without_credentials() { + let store = TestTableCatalogStore::default(); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + + let response = table_bucket_response(&store, "warehouse", true) + .await + .expect("bucket response should build"); + + assert_eq!(response.table_bucket, "warehouse"); + assert!(response.enabled); + assert_eq!(response.catalog_type, crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE); + assert_eq!(response.warehouse_location, "s3://warehouse/"); + assert_eq!(response.catalog_uri, "/iceberg/v1/warehouse"); + assert_eq!(response.compat_catalog_uri, "/_iceberg/v1/warehouse"); + assert_eq!(response.credential_vending, CREDENTIAL_VENDING_UNSUPPORTED); + assert_eq!(response.credential_scope, "warehouse-prefix"); + assert_eq!(response.credential_scope_prefix, "s3://warehouse/"); + assert!(response.catalog_entry_present); +} + +#[test] +fn table_catalog_ingress_requests_reject_unknown_fields() { + assert_rejects_unknown_field::( + "CreateNamespaceRequest", + serde_json::json!({ + "namespace": ["analytics"], + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "RegisterTableRequest", + serde_json::json!({ + "name": "events", + "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "CreateTableRequest", + serde_json::json!({ + "name": "events", + "schema": {}, + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "RestCommitTableRequest", + serde_json::json!({ + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "TableMetadataMaintenanceRequest", + serde_json::json!({ + "delete": true, + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "UpdateTableMetadataLocationRequest", + serde_json::json!({ + "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json", + "version-token": "token-v1", + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "CatalogImportRequest", + serde_json::json!({ + "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "ExternalCatalogBridgeRequest", + serde_json::json!({ + "catalog": "polaris", + "external-namespace": "analytics", + "external-table": "events", + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "ExternalCatalogBridgeSyncRequest", + serde_json::json!({ + "catalog": "glue", + "external-namespace": "analytics", + "external-table": "events", + "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "RollbackTableRequest", + serde_json::json!({ + "metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", + "version-token": "token-v2", + "unexpected": true + }), + ); + assert_rejects_unknown_field::( + "TableMaintenanceConfig", + serde_json::json!({ + "version": 1, + "unexpected": true + }), + ); +} + +fn assert_rejects_unknown_field(target: &str, value: serde_json::Value) +where + T: serde::de::DeserializeOwned, +{ + let err = match serde_json::from_value::(value) { + Ok(_) => panic!("{target} should reject unknown fields"), + Err(err) => err, + }; + assert!( + err.to_string().contains("unknown field"), + "{target} should reject unknown fields, got: {err}" + ); +} + +#[test] +fn create_namespace_request_uses_rest_namespace_segments_and_properties() { + let request: CreateNamespaceRequest = serde_json::from_value(serde_json::json!({ + "namespace": ["analytics", "daily_events"], + "properties": { + "owner": "lakehouse" + } + })) + .expect("request should parse"); + let namespace = namespace_from_segments(&request.namespace).expect("namespace should be valid"); + let response = namespace_response_from_entry(crate::table_catalog::NamespaceEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: namespace.public_name(), + namespace_id: namespace.storage_id(), + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: request.properties, + created_at: None, + updated_at: None, + }) + .expect("namespace response should build"); + + assert_eq!(namespace.public_name(), "analytics.daily_events"); + assert_eq!(response.namespace, vec!["analytics".to_string(), "daily_events".to_string()]); + assert_eq!(response.properties.get("owner").map(String::as_str), Some("lakehouse")); +} + +#[test] +fn list_tables_response_uses_rest_identifier_shape() { + let namespace = crate::table_catalog::Namespace::parse("analytics.daily_events").expect("namespace should parse"); + let response = list_tables_response_from_entries( + vec![crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: namespace.public_name(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" + .to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }], + None, + ) + .expect("table list response should build"); + + assert_eq!( + response.identifiers[0].namespace, + vec!["analytics".to_string(), "daily_events".to_string()] + ); + assert_eq!(response.identifiers[0].name, "events"); + assert!(response.next_page_token.is_none()); +} + +#[test] +fn rest_pagination_round_trips_context_bound_tokens() { + let context = RestPageContext { + resource: TABLE_CATALOG_TABLE_RESOURCE_ROOT, + warehouse: "warehouse", + namespace: Some("analytics"), + }; + let first_request = "/?pageSize=2".parse::().expect("first page URI should parse"); + let first_pagination = rest_pagination_from_query(&first_request, context).expect("pageSize should start pagination"); + let (cursor, limit) = first_pagination.page_request().expect("pageSize should enable pagination"); + assert_eq!(cursor, None); + assert_eq!(limit.get(), 2); + let next_page_token = first_pagination + .next_page_token(Some("strong:beta".to_string())) + .expect("page token should encode") + .expect("page token should be present"); + let second_request = format!("/?pageSize=2&pageToken={next_page_token}") + .parse::() + .expect("second page URI should parse"); + let second_pagination = rest_pagination_from_query(&second_request, context).expect("continuation token should decode"); + let (cursor, limit) = second_pagination + .page_request() + .expect("continuation should remain paginated"); + assert_eq!(cursor, Some("strong:beta")); + assert_eq!(limit.get(), 2); + assert!( + second_pagination + .next_page_token(None) + .expect("terminal token should build") + .is_none() + ); + + let default_size_request = format!("/?pageToken={next_page_token}") + .parse::() + .expect("continuation URI should parse without pageSize"); + let default_size_pagination = + rest_pagination_from_query(&default_size_request, context).expect("continuation should use the default page size"); + let (cursor, limit) = default_size_pagination + .page_request() + .expect("continuation should remain paginated"); + assert_eq!(cursor, Some("strong:beta")); + assert_eq!(limit.get(), REST_DEFAULT_PAGE_SIZE); + + for other_context in [ + RestPageContext { + resource: TABLE_CATALOG_VIEW_RESOURCE_ROOT, + warehouse: "warehouse", + namespace: Some("analytics"), + }, + RestPageContext { + resource: TABLE_CATALOG_TABLE_RESOURCE_ROOT, + warehouse: "other-warehouse", + namespace: Some("analytics"), + }, + RestPageContext { + resource: TABLE_CATALOG_TABLE_RESOURCE_ROOT, + warehouse: "warehouse", + namespace: Some("other-namespace"), + }, + ] { + let expected_context = rest_page_context_fingerprint(other_context); + let error = decode_rest_page_token(&next_page_token, &expected_context).expect_err("cross-context token should fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); + } +} + +#[test] +fn rest_pagination_rejects_invalid_query_parameters() { + let context = RestPageContext { + resource: TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT, + warehouse: "warehouse", + namespace: None, + }; + for uri in [ + "/?pageSize=0", + "/?pageSize=one", + "/?pageSize=1&pageSize=2", + "/?pageToken=first&pageToken=second", + ] { + let uri = uri.parse::().expect("invalid pagination URI should still parse"); + let error = rest_pagination_from_query(&uri, context).expect_err("invalid pagination query should fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into()), "{uri}"); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST), "{uri}"); + } + + let oversized_token = "a".repeat(REST_PAGE_TOKEN_MAX_LENGTH + 1); + let oversized_uri = format!("/?pageToken={oversized_token}") + .parse::() + .expect("oversized token URI should parse"); + let error = rest_pagination_from_query(&oversized_uri, context).expect_err("oversized token should fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); + + let empty_token = "/?pageToken=".parse::().expect("empty token URI should parse"); + let pagination = rest_pagination_from_query(&empty_token, context).expect("empty token should start the first page"); + let (cursor, limit) = pagination.page_request().expect("empty token should enable pagination"); + assert_eq!(cursor, None); + assert_eq!(limit.get(), REST_DEFAULT_PAGE_SIZE); + + let capped_size = "/?pageSize=5000" + .parse::() + .expect("large page size URI should parse"); + let pagination = rest_pagination_from_query(&capped_size, context).expect("large page size should be capped"); + let (cursor, limit) = pagination.page_request().expect("pageSize alone should enable pagination"); + assert_eq!(cursor, None); + assert_eq!(limit.get(), REST_MAX_PAGE_SIZE); + + let unpaginated = rest_pagination_from_query(&"/".parse().expect("URI should parse"), context) + .expect("request without pagination should parse"); + assert!(unpaginated.page_request().is_none()); +} + +#[test] +fn rest_pagination_rejects_malformed_token_payloads() { + let context = RestPageContext { + resource: TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT, + warehouse: "warehouse", + namespace: None, + }; + let context_fingerprint = rest_page_context_fingerprint(context); + let encoded_json = + |value: serde_json::Value| base64_encode_url_safe_no_pad(&serde_json::to_vec(&value).expect("test token should encode")); + let malformed_tokens = [ + "*".to_string(), + base64_encode_url_safe_no_pad(b"not-json"), + encoded_json(serde_json::json!({ + "version": REST_PAGE_TOKEN_VERSION, + "context": context_fingerprint, + "cursor": "strong:alpha", + "unknown": true + })), + encoded_json(serde_json::json!({ + "version": REST_PAGE_TOKEN_VERSION + 1, + "context": context_fingerprint, + "cursor": "strong:alpha" + })), + encoded_json(serde_json::json!({ + "version": REST_PAGE_TOKEN_VERSION, + "context": context_fingerprint, + "cursor": "" + })), + ]; + + for token in malformed_tokens { + let uri = format!( + "/?pageToken={}", + url::form_urlencoded::byte_serialize(token.as_bytes()).collect::() + ) + .parse::() + .expect("malformed token URI should parse"); + let error = rest_pagination_from_query(&uri, context).expect_err("malformed token should fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); + } +} + +#[tokio::test] +async fn rest_list_pagination_covers_namespaces_tables_and_views() { + let store = TestTableCatalogStore::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + for name in ["beta", "alpha"] { + store.namespaces.lock().await.push(crate::table_catalog::NamespaceEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: name.to_string(), + namespace_id: name.to_string(), + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }); + store.tables.lock().await.push(crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: namespace.public_name(), + table: name.to_string(), + table_id: format!("table-{name}"), + table_uuid: format!("table-uuid-{name}"), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: format!("s3://warehouse/tables/table-{name}"), + metadata_location: format!("s3://warehouse/tables/table-{name}/metadata/00001.metadata.json"), + version_token: "token-v1".to_string(), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }); + store.views.lock().await.push(crate::table_catalog::ViewEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: namespace.public_name(), + view: name.to_string(), + view_id: format!("view-{name}"), + view_uuid: format!("view-uuid-{name}"), + format: "ICEBERG_VIEW".to_string(), + format_version: 1, + warehouse_location: format!("s3://warehouse/views/view-{name}"), + metadata_location: format!("s3://warehouse/views/view-{name}/metadata/00001.view.json"), + version_token: "token-v1".to_string(), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }); + } + + let first_uri = "/?pageSize=1".parse::().expect("first page URI should parse"); + let namespaces = list_namespaces_response(&store, "warehouse", &first_uri) + .await + .expect("namespace first page should load"); + assert_eq!(namespaces.namespaces, vec![vec!["alpha".to_string()]]); + let namespace_token = namespaces.next_page_token.expect("namespace continuation should exist"); + let namespace_uri = format!("/?pageSize=1&pageToken={namespace_token}") + .parse::() + .expect("namespace continuation URI should parse"); + let namespaces = list_namespaces_response(&store, "warehouse", &namespace_uri) + .await + .expect("namespace second page should load"); + assert_eq!(namespaces.namespaces, vec![vec!["beta".to_string()]]); + assert!(namespaces.next_page_token.is_none()); + + let tables = list_tables_response(&store, "warehouse", &namespace, &first_uri) + .await + .expect("table first page should load"); + assert_eq!(tables.identifiers.len(), 1); + assert_eq!(tables.identifiers[0].name, "alpha"); + let table_token = tables.next_page_token.expect("table continuation should exist"); + let table_uri = format!("/?pageSize=1&pageToken={table_token}") + .parse::() + .expect("table continuation URI should parse"); + let tables = list_tables_response(&store, "warehouse", &namespace, &table_uri) + .await + .expect("table second page should load"); + assert_eq!(tables.identifiers.len(), 1); + assert_eq!(tables.identifiers[0].name, "beta"); + assert!(tables.next_page_token.is_none()); + + let views = list_views_response(&store, "warehouse", &namespace, &first_uri) + .await + .expect("view first page should load"); + assert_eq!(views.identifiers.len(), 1); + assert_eq!(views.identifiers[0].name, "alpha"); + let view_token = views.next_page_token.expect("view continuation should exist"); + let view_uri = format!("/?pageSize=1&pageToken={view_token}") + .parse::() + .expect("view continuation URI should parse"); + let views = list_views_response(&store, "warehouse", &namespace, &view_uri) + .await + .expect("view second page should load"); + assert_eq!(views.identifiers.len(), 1); + assert_eq!(views.identifiers[0].name, "beta"); + assert!(views.next_page_token.is_none()); + + for uri in ["/", "/?pageSize=2"] { + let uri = uri.parse::().expect("list URI should parse"); + let namespaces = list_namespaces_response(&store, "warehouse", &uri) + .await + .expect("namespace exact page should load"); + let tables = list_tables_response(&store, "warehouse", &namespace, &uri) + .await + .expect("table exact page should load"); + let views = list_views_response(&store, "warehouse", &namespace, &uri) + .await + .expect("view exact page should load"); + assert_eq!(namespaces.namespaces.len(), 2); + assert!(namespaces.next_page_token.is_none()); + assert_eq!(tables.identifiers.len(), 2); + assert!(tables.next_page_token.is_none()); + assert_eq!(views.identifiers.len(), 2); + assert!(views.next_page_token.is_none()); + } +} + +#[test] +fn list_responses_expose_null_next_page_token_at_end() { + for value in [ + serde_json::to_value(RestListNamespacesResponse { + namespaces: vec![vec!["analytics".to_string()]], + next_page_token: None, + }) + .expect("namespace response should serialize"), + serde_json::to_value(RestListTablesResponse { + identifiers: Vec::new(), + next_page_token: None, + }) + .expect("table response should serialize"), + serde_json::to_value(RestListViewsResponse { + identifiers: Vec::new(), + next_page_token: None, + }) + .expect("view response should serialize"), + ] { + assert!(value.get("next-page-token").is_some_and(serde_json::Value::is_null)); + } +} + +#[tokio::test] +async fn namespace_exists_status_uses_head_rest_semantics() { + let store = TestTableCatalogStore::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + + assert_eq!( + namespace_exists_status(&store, "warehouse", &namespace) + .await + .expect("missing namespace check should succeed"), + StatusCode::NOT_FOUND + ); + + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + + assert_eq!( + namespace_exists_status(&store, "warehouse", &namespace) + .await + .expect("existing namespace check should succeed"), + StatusCode::NO_CONTENT + ); +} + +#[tokio::test] +async fn table_exists_status_uses_head_rest_semantics() { + let store = TestTableCatalogStore::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + + assert_eq!( + table_exists_status(&store, "warehouse", &namespace, "events") + .await + .expect("missing table check should succeed"), + StatusCode::NOT_FOUND + ); + + store + .create_table(crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: namespace.public_name(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json" + .to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) + .await + .expect("table should be created"); + + assert_eq!( + table_exists_status(&store, "warehouse", &namespace, "events") + .await + .expect("existing table check should succeed"), + StatusCode::NO_CONTENT + ); +} + +#[test] +fn register_table_request_builds_initial_table_entry() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: RegisterTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "metadata-location": "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", + "overwrite": false + })) + .expect("request should parse"); + + let entry = table_entry_from_register_request("warehouse", &namespace, request).expect("table entry should build"); + + assert_eq!(entry.table_bucket, "warehouse"); + assert_eq!(entry.namespace, "analytics"); + assert_eq!(entry.table, "events"); + assert_eq!( + entry.metadata_location, + ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json" + ); + assert!(entry.properties.is_empty()); + assert_eq!(entry.generation, 1); + assert!(!entry.version_token.is_empty()); +} + +#[test] +fn create_table_request_accepts_standard_iceberg_rest_shape() { + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + } + ] + }, + "partition-spec": { + "spec-id": 0, + "fields": [] + }, + "write-order": { + "order-id": 0, + "fields": [] + }, + "properties": { + "write.format.default": "parquet" + } + })) + .expect("standard create table request should parse"); + + assert_eq!(request.name, "events"); +} + +#[test] +fn commit_table_request_accepts_standard_iceberg_rest_shape() { + let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-table-uuid", + "uuid": "table-uuid" + } + ], + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "lakehouse" + } + } + ] + })) + .expect("standard commit table request should parse"); + + assert_eq!(request.requirements.len(), 1); +} + +#[test] +fn standard_commit_ids_use_uuid_for_metadata_file_when_provided() { + let commit_id = "11111111-1111-4111-8111-111111111111"; + assert_eq!( + standard_commit_ids(Some(commit_id.to_string())), + (commit_id.to_string(), commit_id.to_string()) + ); +} + +#[test] +fn standard_commit_ids_generate_metadata_hash_for_non_uuid_client_id() { + let (commit_id, metadata_file_token) = standard_commit_ids(Some("commit-1".to_string())); + + assert_eq!(commit_id, "commit-1"); + assert_ne!(metadata_file_token, commit_id); + assert_eq!(metadata_file_token, table_catalog_path_hash("commit-1")); +} + +#[tokio::test] +async fn create_table_response_writes_initial_metadata_for_standard_request() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + } + ] + }, + "properties": { + "write.format.default": "parquet" + } + })) + .expect("standard create table request should parse"); + + let response = create_table_response(&store, &metadata_backend, "warehouse", &namespace, request, true) + .await + .expect("table should be created"); + + assert_eq!(response.metadata["format-version"], 2); + assert_eq!(response.metadata["current-schema-id"], 0); + assert_eq!(response.metadata["default-spec-id"], 0); + assert_eq!(response.metadata["default-sort-order-id"], 0); + assert_eq!( + response.metadata["properties"]["write.format.default"], + serde_json::Value::String("parquet".to_string()) + ); + let entry = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + assert_eq!( + response.metadata_location, + format!( + "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001-{}.metadata.json", + entry.table_id + ) + ); + assert_eq!(response.metadata["table-uuid"], entry.table_uuid); + assert!( + metadata_backend + .object_exists("warehouse", &entry.metadata_location) + .await + .expect("metadata object lookup should succeed") + ); +} + +#[tokio::test] +async fn create_table_response_recreates_dropped_identifier_without_overwriting_retained_metadata() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + let first_entry = store + .load_table("warehouse", "analytics", "events") + .await + .expect("first table lookup should succeed") + .expect("first table should exist"); + let retained_initial = metadata_backend + .read_object("warehouse", &first_entry.metadata_location) + .await + .expect("first metadata lookup should succeed") + .expect("first metadata should exist"); + + let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "first-table" + } + } + ] + })) + .expect("standard commit request should parse"); + standard_commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) + .await + .expect("first table metadata commit should succeed"); + let first_committed_entry = store + .load_table("warehouse", "analytics", "events") + .await + .expect("committed table lookup should succeed") + .expect("committed table should exist"); + + drop_table_in_store(&store, "warehouse", &namespace, "events") + .await + .expect("first table should drop"); + drop_namespace_in_store(&store, "warehouse", "analytics") + .await + .expect("first namespace should drop"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be recreated"); + + let create_request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + } + ] + } + })) + .expect("recreate table request should parse"); + let second = create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request, true) + .await + .expect("dropped table identifier should be reusable"); + let second_entry = store + .load_table("warehouse", "analytics", "events") + .await + .expect("recreated table lookup should succeed") + .expect("recreated table should exist"); + + assert_ne!(first_entry.table_id, second_entry.table_id); + assert_ne!(first_entry.table_uuid, second_entry.table_uuid); + assert_ne!(first_entry.metadata_location, second_entry.metadata_location); + assert_ne!(first_committed_entry.metadata_location, second_entry.metadata_location); + assert_eq!(second.metadata["table-uuid"], second_entry.table_uuid); + assert_eq!(second.metadata["metadata-log"], serde_json::json!([])); + assert_eq!( + metadata_backend + .read_object("warehouse", &first_entry.metadata_location) + .await + .expect("retained metadata lookup should succeed") + .expect("retained metadata should still exist") + .data, + retained_initial.data + ); + assert!( + metadata_backend + .object_exists("warehouse", &first_committed_entry.metadata_location) + .await + .expect("committed metadata lookup should succeed") + ); + assert!( + metadata_backend + .object_exists("warehouse", &second_entry.metadata_location) + .await + .expect("recreated metadata lookup should succeed") + ); +} + +#[tokio::test] +async fn concurrent_create_table_responses_keep_one_catalog_winner_with_distinct_metadata() { + let catalog_backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(catalog_backend); + let metadata_backend = TestTableCatalogObjectBackend { + objects: Arc::new(tokio::sync::Mutex::new(BTreeMap::new())), + put_object_barrier: Some(Arc::new(tokio::sync::Barrier::new(2))), + }; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let create_request = || { + serde_json::from_value::(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + } + ] + } + })) + .expect("concurrent create table request should parse") + }; + + let (first, second) = tokio::join!( + create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request(), true,), + create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request(), true,) + ); + assert_ne!(first.is_ok(), second.is_ok(), "exactly one concurrent create should succeed"); + let (winner, loser) = match (first, second) { + (Ok(winner), Err(loser)) | (Err(loser), Ok(winner)) => (winner, loser), + _ => unreachable!("success count was checked above"), + }; + assert!(format!("{loser:?}").contains("AlreadyExistsException")); + + let tables = store + .list_tables("warehouse", "analytics") + .await + .expect("table listing should succeed"); + assert_eq!(tables.len(), 1); + let winner_entry = &tables[0]; + assert_eq!( + winner.metadata_location, + table_metadata_location_for_client("warehouse", &winner_entry.metadata_location) + ); + let metadata_prefix = winner_entry + .metadata_location + .rsplit_once('/') + .map(|(prefix, _)| format!("{prefix}/")) + .expect("metadata location should contain a file name"); + let metadata_objects = metadata_backend + .list_objects("warehouse", &metadata_prefix) + .await + .expect("metadata listing should succeed"); + assert_eq!(metadata_objects.len(), 2); + assert_ne!(metadata_objects[0], metadata_objects[1]); + + let mut table_uuids = Vec::with_capacity(metadata_objects.len()); + for metadata_object in metadata_objects { + let metadata = metadata_backend + .read_object("warehouse", &metadata_object) + .await + .expect("metadata lookup should succeed") + .expect("metadata object should exist"); + let metadata: serde_json::Value = serde_json::from_slice(&metadata.data).expect("metadata object should contain JSON"); + table_uuids.push( + metadata["table-uuid"] + .as_str() + .expect("metadata should contain table uuid") + .to_string(), + ); + } + table_uuids.sort(); + table_uuids.dedup(); + assert_eq!(table_uuids.len(), 2); + assert!(table_uuids.contains(&winner_entry.table_uuid)); +} + +#[tokio::test] +async fn standard_commit_applies_updates_and_writes_next_metadata() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_uuid = created.metadata["table-uuid"] + .as_str() + .expect("created metadata should have table uuid") + .to_string(); + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + + let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-table-uuid", + "uuid": table_uuid + }, + { + "type": "assert-current-schema-id", + "current-schema-id": 0 + } + ], + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "lakehouse" + } + }, + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("standard commit table request should parse"); + + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) + .await + .expect("standard commit should succeed"); + + let metadata_file_prefix = + "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-"; + let metadata_file_suffix = ".metadata.json"; + let generated_commit_id = commit + .metadata_location + .strip_prefix(metadata_file_prefix) + .and_then(|file| file.strip_suffix(metadata_file_suffix)) + .expect("standard commit metadata file should include a UUID suffix"); + Uuid::parse_str(generated_commit_id).expect("metadata file suffix should be a UUID"); + assert_eq!(commit.commit_id, generated_commit_id); + assert_eq!(commit.metadata["properties"]["owner"], serde_json::Value::String("lakehouse".to_string())); + assert_eq!(commit.metadata["current-snapshot-id"], 10); + assert_eq!(commit.metadata["last-sequence-number"], 1); + assert_eq!(commit.metadata["refs"]["main"]["snapshot-id"], 10); + assert_eq!(commit.metadata["metadata-log"][0]["metadata-file"], created.metadata_location); + let committed = store + .load_table("warehouse", "analytics", "events") + .await + .expect("committed table lookup should succeed") + .expect("committed table should exist"); + assert_eq!( + table_metadata_location_for_client("warehouse", &committed.metadata_location), + commit.metadata_location + ); + assert!( + metadata_backend + .object_exists("warehouse", &committed.metadata_location) + .await + .expect("committed metadata lookup should succeed") + ); +} + +#[tokio::test] +async fn standard_commit_uses_client_uuid_commit_id_in_metadata_file_name() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + + let commit_id = "11111111-1111-4111-8111-111111111111"; + let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": commit_id, + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "lakehouse" + } + } + ] + })) + .expect("standard commit table request should parse"); + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) + .await + .expect("standard commit should succeed"); + + assert_eq!(commit.commit_id, commit_id); + assert_eq!( + commit.metadata_location, + "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-11111111-1111-4111-8111-111111111111.metadata.json" + ); + let committed = store + .load_table("warehouse", "analytics", "events") + .await + .expect("committed table lookup should succeed") + .expect("committed table should exist"); + assert!( + metadata_backend + .object_exists("warehouse", &committed.metadata_location) + .await + .expect("committed metadata lookup should succeed") + ); +} + +#[tokio::test] +async fn standard_commit_accepts_non_uuid_client_commit_id_without_using_it_in_metadata_file_name() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + + let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": "commit-1", + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "lakehouse" + } + } + ] + })) + .expect("standard commit table request should parse"); + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) + .await + .expect("standard commit should succeed"); + + let metadata_file_prefix = + "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-"; + let metadata_file_suffix = ".metadata.json"; + let metadata_file_token = commit + .metadata_location + .strip_prefix(metadata_file_prefix) + .and_then(|file| file.strip_suffix(metadata_file_suffix)) + .expect("standard commit metadata file should include a safe token suffix"); + assert_eq!(commit.commit_id, "commit-1"); + assert_ne!(metadata_file_token, commit.commit_id); + assert_eq!(metadata_file_token, table_catalog_path_hash("commit-1")); +} + +#[tokio::test] +async fn standard_commit_ignores_generation_only_orphan_metadata_file() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + metadata_backend + .put_json( + "warehouse", + ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json", + serde_json::json!({ + "format-version": 2, + "table-uuid": "orphan", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + let commit_id = "22222222-2222-4222-8222-222222222222"; + let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": commit_id, + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "lakehouse" + } + } + ] + })) + .expect("standard commit table request should parse"); + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) + .await + .expect("standard commit should not collide with generation-only orphan"); + + assert_eq!( + commit.metadata_location, + "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-22222222-2222-4222-8222-222222222222.metadata.json" + ); + assert_eq!(commit.metadata["properties"]["owner"], "lakehouse"); +} + +#[tokio::test] +async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointer_conflict() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let metadata_backend = TestTableCatalogObjectBackend { + objects: Arc::clone(&metadata_backend.objects), + put_object_barrier: Some(barrier), + }; + let first_commit_id = "33333333-3333-4333-8333-333333333333"; + let second_commit_id = "44444444-4444-4444-8444-444444444444"; + let first_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": first_commit_id, + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "first" + } + } + ] + })) + .expect("first standard commit table request should parse"); + let second_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": second_commit_id, + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "second" + } + } + ] + })) + .expect("second standard commit table request should parse"); + + let (first, second) = tokio::join!( + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_request), + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_request) + ); + let success_count = [first.is_ok(), second.is_ok()].into_iter().filter(|ok| *ok).count(); + + assert_eq!(success_count, 1); + assert!( + metadata_backend + .object_exists( + "warehouse", + ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-33333333-3333-4333-8333-333333333333.metadata.json" + ) + .await + .expect("first metadata object lookup should succeed") + ); + assert!( + metadata_backend + .object_exists( + "warehouse", + ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-44444444-4444-4444-8444-444444444444.metadata.json" + ) + .await + .expect("second metadata object lookup should succeed") + ); +} + +#[tokio::test] +async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_matches() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + let legacy_entry = table_entry_from_register_request( + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: current_location.to_string(), + overwrite: false, + }, + ) + .expect("table entry should build"); + assert_ne!(legacy_entry.table_uuid, "metadata-table-uuid"); + store + .register_table(legacy_entry.clone()) + .await + .expect("legacy table entry should register"); + metadata_backend + .put_json( + "warehouse", + current_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "metadata-table-uuid", + "location": "s3://warehouse/tables/table-id", + "properties": {} + }), + ) + .await; + + let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "set-properties", + "updates": { + "owner": "lakehouse" + } + } + ] + })) + .expect("standard commit table request should parse"); + let committed = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) + .await + .expect("legacy catalog uuid should not block standard commit"); + + assert_eq!(committed.metadata["table-uuid"], "metadata-table-uuid"); + assert_eq!(committed.metadata["properties"]["owner"], "lakehouse"); + assert_eq!(committed.generation, legacy_entry.generation + 1); +} + +#[tokio::test] +async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_current_metadata() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + let legacy_entry = table_entry_from_register_request( + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: current_location.to_string(), + overwrite: false, + }, + ) + .expect("table entry should build"); + assert_ne!(legacy_entry.table_uuid, "metadata-table-uuid"); + store + .register_table(legacy_entry.clone()) + .await + .expect("legacy table entry should register"); + metadata_backend + .put_json( + "warehouse", + current_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "metadata-table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + let next_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; + metadata_backend + .put_json( + "warehouse", + next_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "metadata-table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-sequence-number": 2 + }), + ) + .await; + + let updated = update_table_metadata_location_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "events", + UpdateTableMetadataLocationRequest { + metadata_location: next_location.to_string(), + version_token: legacy_entry.version_token, + commit_id: Some("commit-1".to_string()), + idempotency_key: None, + }, + ) + .await + .expect("legacy catalog uuid should not block metadata-location update"); + + assert_eq!(updated.metadata_location, table_metadata_location_for_client("warehouse", next_location)); + assert_eq!(updated.generation, legacy_entry.generation + 1); +} + +#[tokio::test] +async fn table_metadata_maintenance_helper_runs_dry_run_and_delete() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let old = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; + backend + .put_json_with_mod_time(bucket, &old, serde_json::json!({}), Some(OffsetDateTime::UNIX_EPOCH)) + .await; + backend + .put_json_with_mod_time( + bucket, + ¤t, + serde_json::json!({ + "current-snapshot-id": 20, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "timestamp-ms": 1000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro" + }, + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-20.avro" + } + ], + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + }), + Some(OffsetDateTime::UNIX_EPOCH), + ) + .await; + + let default_config = store + .get_table_maintenance_config(bucket, "analytics", "events") + .await + .expect("default maintenance config should load"); + assert_eq!(default_config, crate::table_catalog::TableMaintenanceConfig::default()); + let config = store + .put_table_maintenance_config( + bucket, + "analytics", + "events", + crate::table_catalog::TableMaintenanceConfig { + version: crate::table_catalog::TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 2, + delete_enabled: true, + background_enabled: false, + ..Default::default() + }, + ) + .await + .expect("maintenance config should persist"); + assert_eq!(config.retain_recent_metadata_files, 2); + assert!(config.delete_enabled); + let background_config = store + .put_table_maintenance_config( + bucket, + "analytics", + "events", + crate::table_catalog::TableMaintenanceConfig { + version: crate::table_catalog::TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 2, + delete_enabled: true, + background_enabled: true, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + assert!(background_config.background_enabled); + + let dry_run = table_metadata_maintenance_response( + &store, + &backend, + bucket, + &namespace, + "events", + TableMetadataMaintenanceRequest { + retain_recent_metadata_files: 0, + delete: false, + snapshot_expiration: Some(crate::table_catalog::TableSnapshotExpirationConfig { + min_snapshots_to_keep: 1, + max_snapshot_age_ms: 1, + }), + commit_snapshot_expiration: false, + compaction: Some(crate::table_catalog::TableCompactionPlanningConfig { + target_file_size_bytes: 512 * 1024 * 1024, + small_file_threshold_bytes: 64 * 1024 * 1024, + min_input_files: 2, + max_rewrite_bytes_per_job: 1024 * 1024 * 1024, + }), + commit_compaction: false, + }, + ) + .await + .expect("metadata maintenance dry-run should succeed"); + assert_eq!(dry_run.cleanup_candidate_locations, vec![old.clone()]); + assert_eq!(dry_run.deletable_metadata_locations, vec![old.clone()]); + let snapshot_expiration = dry_run + .snapshot_expiration + .as_ref() + .expect("dry-run report should include snapshot expiration planning"); + assert_eq!(snapshot_expiration.expiration_candidate_count, 1); + assert_eq!(snapshot_expiration.current_snapshot_id, Some(20)); + let compaction = dry_run + .compaction + .as_ref() + .expect("dry-run report should include compaction planning"); + assert_eq!( + compaction.status, + crate::table_catalog::TableCompactionPlanningStatus::ManualReviewRequired + ); + assert_eq!(compaction.manual_review_count, 1); + let stored_dry_run = store + .get_table_metadata_maintenance_report(bucket, "analytics", "events", &dry_run.job.job_id) + .await + .expect("maintenance job lookup should succeed") + .expect("maintenance job should be stored"); + assert_eq!(stored_dry_run, dry_run); + assert!( + backend + .object_exists(bucket, &old) + .await + .expect("old metadata lookup should succeed") + ); + + let deleted = table_metadata_maintenance_response( + &store, + &backend, + bucket, + &namespace, + "events", + TableMetadataMaintenanceRequest { + retain_recent_metadata_files: 0, + delete: true, + snapshot_expiration: None, + commit_snapshot_expiration: false, + compaction: None, + commit_compaction: false, + }, + ) + .await + .expect("metadata maintenance delete should succeed"); + assert_eq!(deleted.cleanup_candidate_locations, vec![old.clone()]); + assert_eq!(deleted.deletable_metadata_locations, vec![old.clone()]); + assert!( + !backend + .object_exists(bucket, &old) + .await + .expect("old metadata lookup should succeed after delete") + ); +} + +#[tokio::test] +async fn table_metadata_maintenance_helper_commits_snapshot_expiration() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; + backend + .put_json_with_mod_time( + bucket, + ¤t, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-sequence-number": 2, + "last-updated-ms": 2000, + "last-column-id": 1, + "schemas": [], + "current-schema-id": 0, + "partition-specs": [], + "default-spec-id": 0, + "sort-orders": [], + "default-sort-order-id": 0, + "current-snapshot-id": 20, + "metadata-log": [], + "snapshot-log": [ + { + "timestamp-ms": 1000, + "snapshot-id": 10 + }, + { + "timestamp-ms": 2000, + "snapshot-id": 20 + } + ], + "snapshots": [ + { + "snapshot-id": 10, + "timestamp-ms": 1000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro" + }, + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-20.avro" + } + ], + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + }), + Some(OffsetDateTime::UNIX_EPOCH), + ) + .await; + + let report = table_metadata_maintenance_response( + &store, + &backend, + bucket, + &namespace, + "events", + TableMetadataMaintenanceRequest { + retain_recent_metadata_files: 0, + delete: false, + snapshot_expiration: Some(crate::table_catalog::TableSnapshotExpirationConfig { + min_snapshots_to_keep: 1, + max_snapshot_age_ms: 1, + }), + commit_snapshot_expiration: true, + compaction: None, + commit_compaction: false, + }, + ) + .await + .expect("snapshot expiration commit should succeed"); + + let snapshot_expiration = report + .snapshot_expiration + .as_ref() + .expect("maintenance report should include snapshot expiration"); + assert_eq!(snapshot_expiration.expired_snapshot_ids, vec![10]); + let committed_location = snapshot_expiration + .committed_metadata_location + .as_ref() + .expect("snapshot expiration commit should report committed metadata") + .clone(); + assert_ne!(committed_location, current); + + let entry = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + assert_eq!(entry.metadata_location, committed_location); + assert_eq!(entry.generation, 2); + + let committed_object = backend + .read_object(bucket, &entry.metadata_location) + .await + .expect("committed metadata lookup should succeed") + .expect("committed metadata object should exist"); + let committed_metadata = + serde_json::from_slice::(&committed_object.data).expect("committed metadata should be valid JSON"); + let snapshots = committed_metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .expect("committed metadata should contain snapshots"); + assert_eq!(snapshots.len(), 1); + assert_eq!(snapshots[0].get("snapshot-id").and_then(serde_json::Value::as_i64), Some(20)); + let snapshot_log = committed_metadata + .get("snapshot-log") + .and_then(serde_json::Value::as_array) + .expect("committed metadata should contain snapshot-log"); + assert_eq!(snapshot_log.len(), 1); + assert_eq!( + committed_metadata["metadata-log"][0]["metadata-file"], + serde_json::Value::String(table_metadata_location_for_client(bucket, ¤t)) + ); + assert!( + backend + .object_exists(bucket, ¤t) + .await + .expect("previous metadata lookup should succeed") + ); +} + +#[tokio::test] +async fn table_metadata_maintenance_helper_rejects_snapshot_expiration_manual_review_commit() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; + backend + .put_json_with_mod_time( + bucket, + ¤t, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "current-snapshot-id": 20, + "metadata-log": [], + "snapshot-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "timestamp-ms": 1000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro" + }, + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-20.avro" + } + ], + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + }, + "audit": { + "snapshot-id": 10, + "type": "tag" + } + } + }), + Some(OffsetDateTime::UNIX_EPOCH), + ) + .await; + + let result = table_metadata_maintenance_response( + &store, + &backend, + bucket, + &namespace, + "events", + TableMetadataMaintenanceRequest { + retain_recent_metadata_files: 0, + delete: false, + snapshot_expiration: Some(crate::table_catalog::TableSnapshotExpirationConfig { + min_snapshots_to_keep: 1, + max_snapshot_age_ms: 1, + }), + commit_snapshot_expiration: true, + compaction: None, + commit_compaction: false, + }, + ) + .await; + + assert!(result.is_err()); + let entry = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + assert_eq!(entry.metadata_location, current); + assert_eq!(entry.generation, 1); +} + +#[tokio::test] +async fn table_metadata_maintenance_helper_rejects_stale_snapshot_expiration_plan() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let next = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + let metadata = serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "current-snapshot-id": 20, + "metadata-log": [], + "snapshot-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "timestamp-ms": 1000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro" + }, + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-20.avro" + } + ], + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + }); + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; + backend + .put_json_with_mod_time(bucket, ¤t, metadata.clone(), Some(OffsetDateTime::UNIX_EPOCH)) + .await; + backend + .put_json_with_mod_time(bucket, &next, metadata, Some(OffsetDateTime::UNIX_EPOCH)) + .await; + + let stale_plan = store + .plan_table_snapshot_expiration( + bucket, + "analytics", + "events", + crate::table_catalog::TableSnapshotExpirationConfig { + min_snapshots_to_keep: 1, + max_snapshot_age_ms: 1, + }, + ) + .await + .expect("snapshot expiration plan should build"); + store + .commit_table(crate::table_catalog::TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "advance-pointer".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current, + new_metadata_location: next, + requirements: Vec::new(), + writer: Some("test".to_string()), + }) + .await + .expect("pointer advance should succeed"); + + let result = commit_table_snapshot_expiration_response(&store, &backend, bucket, &namespace, "events", stale_plan).await; + + assert!(result.is_err()); + let entry = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + assert_eq!(entry.generation, 2); +} + +#[tokio::test] +async fn table_metadata_maintenance_helper_rejects_delete_with_snapshot_expiration_commit() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + + let result = table_metadata_maintenance_response( + &store, + &backend, + "warehouse", + &namespace, + "events", + TableMetadataMaintenanceRequest { + retain_recent_metadata_files: 0, + delete: true, + snapshot_expiration: Some(crate::table_catalog::TableSnapshotExpirationConfig { + min_snapshots_to_keep: 1, + max_snapshot_age_ms: 1, + }), + commit_snapshot_expiration: true, + compaction: None, + commit_compaction: false, + }, + ) + .await; + + assert!(result.is_err()); +} + +#[tokio::test] +async fn table_refs_response_reports_current_and_user_defined_refs() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; + backend + .put_json_with_mod_time( + bucket, + ¤t, + serde_json::json!({ + "current-snapshot-id": 10, + "refs": { + "main": { + "snapshot-id": 10, + "type": "branch" + }, + "audit": { + "snapshot-id": 9, + "type": "tag" + } + } + }), + Some(OffsetDateTime::UNIX_EPOCH), + ) + .await; + + let response = table_refs_response(&store, &backend, bucket, &namespace, "events") + .await + .expect("refs response should load"); + + assert_eq!(response.current_snapshot_id, Some(10)); + assert_eq!(response.protected_ref_count, 1); + assert_eq!(response.user_defined_ref_count, 1); + assert!(response.refs.contains_key("main")); + assert!(response.refs.contains_key("audit")); +} + +#[tokio::test] +async fn external_catalog_bridge_response_lists_supported_operator_bridges() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, bucket, true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + bucket, + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let response = external_catalog_bridge_response(&store, bucket, &namespace, "events") + .await + .expect("bridge response should load"); + + assert_eq!(response.status, "bridge-unconfigured"); + assert!(response.unsupported_bridges.is_empty()); + assert_eq!(response.capabilities.len(), 4); + assert!( + response + .capabilities + .iter() + .any(|bridge| bridge.catalog == "polaris" && bridge.status == "operator-sync-supported") + ); +} + +#[tokio::test] +async fn external_catalog_bridge_persists_identity_and_boundary() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, bucket, true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + bucket, + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + + let configured = put_external_catalog_bridge_response( + &store, + bucket, + &namespace, + "events", + ExternalCatalogBridgeRequest { + catalog: "polaris".to_string(), + external_catalog_id: Some("prod-catalog".to_string()), + external_namespace: "sales.analytics".to_string(), + external_table: "orders".to_string(), + external_table_uuid: Some("table-uuid".to_string()), + metadata_location: None, + external_version_token: Some("polaris-v7".to_string()), + policy_mode: Some("rustfs-authoritative".to_string()), + credential_mode: Some("rustfs-table-credentials".to_string()), + sync_mode: Some("manual".to_string()), + properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), + }, + ) + .await + .expect("bridge should configure"); + + assert_eq!(configured.status, "bridge-configured"); + let bridge = configured.bridge.as_ref().expect("bridge state should be returned"); + assert_eq!(bridge.catalog, "polaris"); + assert_eq!(bridge.external_catalog_id.as_deref(), Some("prod-catalog")); + assert_eq!(bridge.external_namespace, "sales.analytics"); + assert_eq!(bridge.external_table, "orders"); + assert_eq!(bridge.external_version_token.as_deref(), Some("polaris-v7")); + assert_eq!(bridge.policy_mode, "rustfs-authoritative"); + assert_eq!(bridge.credential_mode, "rustfs-table-credentials"); + assert_eq!(bridge.sync_mode, "manual"); + + let loaded = external_catalog_bridge_response(&store, bucket, &namespace, "events") + .await + .expect("bridge response should load"); + assert_eq!(loaded.status, "bridge-configured"); + assert!( + loaded + .capabilities + .iter() + .any(|capability| capability.catalog == "polaris" && capability.status == "operator-sync-supported") + ); +} + +#[tokio::test] +async fn external_catalog_bridge_sync_registers_missing_table_from_snapshot() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + ensure_table_bucket_entry(&store, bucket, true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + bucket, + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let metadata_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + backend + .put_json( + bucket, + &metadata_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + let synced = sync_external_catalog_bridge_response( + &store, + &backend, + bucket, + &namespace, + "events", + ExternalCatalogBridgeSyncRequest { + catalog: "glue".to_string(), + external_catalog_id: Some("aws-glue-prod".to_string()), + external_namespace: "sales".to_string(), + external_table: "orders".to_string(), + external_table_uuid: Some("table-uuid".to_string()), + metadata_location: metadata_location.clone(), + external_version_token: Some("glue-version-1".to_string()), + expected_version_token: None, + expected_metadata_location: None, + commit_id: Some("external-sync-1".to_string()), + idempotency_key: Some("external-sync-idempotency-1".to_string()), + policy_mode: Some("rustfs-authoritative".to_string()), + credential_mode: Some("rustfs-table-credentials".to_string()), + rollback_strategy: Some("retain-current-pointer".to_string()), + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("external sync should register missing table"); + + assert_eq!(synced.action, "registered"); + assert_eq!( + synced.table.metadata_location, + table_metadata_location_for_client(bucket, &metadata_location) + ); + let bridge = synced.bridge.bridge.as_ref().expect("bridge state should be returned"); + assert_eq!(bridge.catalog, "glue"); + assert_eq!(bridge.last_sync_status.as_deref(), Some("synced")); + assert_eq!(bridge.last_synced_metadata_location.as_deref(), Some(metadata_location.as_str())); + assert_eq!(bridge.rollback_strategy, "retain-current-pointer"); + let current = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should be registered"); + assert_eq!(current.metadata_location, metadata_location); + assert_eq!(current.table_uuid, "table-uuid"); +} + +#[tokio::test] +async fn external_catalog_bridge_sync_commits_existing_table_pointer() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current_location.clone()).await; + backend + .put_json( + bucket, + ¤t_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-sequence-number": 1 + }), + ) + .await; + backend + .put_json( + bucket, + &next_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-sequence-number": 2 + }), + ) + .await; + + let synced = sync_external_catalog_bridge_response( + &store, + &backend, + bucket, + &namespace, + "events", + ExternalCatalogBridgeSyncRequest { + catalog: "hive-metastore".to_string(), + external_catalog_id: Some("hms-prod".to_string()), + external_namespace: "sales".to_string(), + external_table: "orders".to_string(), + external_table_uuid: Some("table-uuid".to_string()), + metadata_location: table_metadata_location_for_client(bucket, &next_location), + external_version_token: Some("hms-version-2".to_string()), + expected_version_token: Some("token-v1".to_string()), + expected_metadata_location: Some(table_metadata_location_for_client(bucket, ¤t_location)), + commit_id: Some("external-sync-2".to_string()), + idempotency_key: Some("external-sync-idempotency-2".to_string()), + policy_mode: Some("rustfs-authoritative".to_string()), + credential_mode: Some("rustfs-table-credentials".to_string()), + rollback_strategy: Some("retain-current-pointer".to_string()), + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("external sync should commit existing table"); + + assert_eq!(synced.action, "committed"); + assert_eq!(synced.table.metadata_location, table_metadata_location_for_client(bucket, &next_location)); + let current = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + assert_eq!(current.metadata_location, next_location); + assert_eq!(current.generation, 2); +} + +#[tokio::test] +async fn external_catalog_bridge_sync_conflicts_leave_pointer_unchanged() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current_location.clone()).await; + backend + .put_json( + bucket, + ¤t_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + backend + .put_json( + bucket, + &next_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "different-table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + let result = sync_external_catalog_bridge_response( + &store, + &backend, + bucket, + &namespace, + "events", + ExternalCatalogBridgeSyncRequest { + catalog: "dlf".to_string(), + external_catalog_id: Some("dlf-prod".to_string()), + external_namespace: "sales".to_string(), + external_table: "orders".to_string(), + external_table_uuid: Some("different-table-uuid".to_string()), + metadata_location: next_location.clone(), + external_version_token: Some("dlf-version-2".to_string()), + expected_version_token: Some("token-v1".to_string()), + expected_metadata_location: Some(current_location.clone()), + commit_id: Some("external-sync-3".to_string()), + idempotency_key: Some("external-sync-idempotency-3".to_string()), + policy_mode: Some("rustfs-authoritative".to_string()), + credential_mode: Some("rustfs-table-credentials".to_string()), + rollback_strategy: Some("retain-current-pointer".to_string()), + properties: BTreeMap::new(), + }, + true, + ) + .await; + + assert!(result.is_err()); + let current = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + assert_eq!(current.metadata_location, current_location); + assert_eq!(current.version_token, "token-v1"); +} + +#[test] +fn commit_requirements_reject_mismatched_table_uuid() { + let metadata = serde_json::json!({ + "table-uuid": "actual-table-uuid" + }); + let requirements = vec![serde_json::json!({ + "type": "assert-table-uuid", + "uuid": "stale-table-uuid" + })]; + + assert!(validate_table_commit_requirements(&metadata, &requirements).is_err()); +} + +#[test] +fn snapshot_conflict_requirements_validate_current_snapshot_id() { + let metadata = serde_json::json!({ + "current-snapshot-id": 10 + }); + + let matching = vec![serde_json::json!({ + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + })]; + validate_table_commit_requirements(&metadata, &matching).expect("matching current snapshot should pass"); + + let stale = vec![serde_json::json!({ + "type": "assert-current-snapshot-id", + "snapshot-id": 9 + })]; + assert!(validate_table_commit_requirements(&metadata, &stale).is_err()); + + let no_snapshot_metadata = serde_json::json!({}); + let create_like = vec![serde_json::json!({ + "type": "assert-current-snapshot-id", + "snapshot-id": null + })]; + validate_table_commit_requirements(&no_snapshot_metadata, &create_like) + .expect("null current snapshot requirement should pass when no current snapshot exists"); +} + +#[test] +fn snapshot_conflict_rejects_stale_parent_or_sequence_number() { + let metadata = serde_json::json!({ + "current-snapshot-id": 10, + "last-sequence-number": 4, + "snapshots": [ + { + "snapshot-id": 10, + "sequence-number": 4, + "timestamp-ms": 1234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": { + "operation": "append" + } + } + ], + "snapshot-log": [], + "metadata-log": [] + }); + + let stale_parent = vec![serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 9, + "sequence-number": 5, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": { + "operation": "append" + } + } + })]; + assert!(apply_table_commit_updates(metadata.clone(), &stale_parent, "metadata/00001.metadata.json").is_err()); + + let stale_sequence = vec![serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 4, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": { + "operation": "append" + } + } + })]; + assert!(apply_table_commit_updates(metadata, &stale_sequence, "metadata/00001.metadata.json").is_err()); +} + +#[test] +fn snapshot_conflict_rejects_unknown_snapshot_operations() { + let metadata = serde_json::json!({ + "current-snapshot-id": 10, + "last-sequence-number": 4, + "snapshots": [ + { + "snapshot-id": 10, + "sequence-number": 4, + "timestamp-ms": 1234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": { + "operation": "append" + } + } + ], + "snapshot-log": [], + "metadata-log": [] + }); + + let updates = vec![serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 5, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": { + "operation": "unknown" + } + } + })]; + assert!(apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json").is_err()); +} + +#[tokio::test] +async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let old_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + ¤t_manifest_list, + 10, + 1, + &[(&old_data_file, 0, 1, 10, 1)], + ) + .await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": current_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect("append commit should succeed"); + + let overwrite_manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let replacement_data_file = format!("{table_location}/data/part-11.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + &overwrite_manifest_list, + 11, + 2, + &[(&old_data_file, 0, 2, 11, 2), (&replacement_data_file, 0, 1, 11, 2)], + ) + .await; + let overwrite_request_json = serde_json::json!({ + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": overwrite_manifest_list, + "summary": { + "operation": "overwrite" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + } + ] + }); + let stale_overwrite_request: RestCommitTableRequest = + serde_json::from_value(overwrite_request_json.clone()).expect("stale overwrite request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", stale_overwrite_request) + .await + .expect_err("deleted file sequence must not change"); + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + &overwrite_manifest_list, + 11, + 2, + &[(&old_data_file, 0, 2, 11, 1), (&replacement_data_file, 0, 1, 11, 2)], + ) + .await; + let overwrite_request: RestCommitTableRequest = + serde_json::from_value(overwrite_request_json).expect("overwrite request should parse"); + + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) + .await + .expect("overwrite commit should pass manifest conflict validation"); + + assert_eq!(commit.metadata["current-snapshot-id"], 11); + assert_eq!(commit.metadata["last-sequence-number"], 2); +} + +#[tokio::test] +async fn row_level_conflict_allows_v1_manifest_snapshot() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest = format!("{table_location}/metadata/manifest-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, 1)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifests": [ + manifest.clone() + ], + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect("v1 manifests snapshot should commit"); + + assert_eq!(commit.metadata["current-snapshot-id"], 10); + assert_eq!(commit.metadata["last-sequence-number"], 1); + + let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); + let second_data_file = format!("{table_location}/data/part-11.parquet"); + let second_manifest_list_key = test_snapshot_object_key("warehouse", &second_manifest_list); + metadata_backend + .put_bytes( + "warehouse", + &second_manifest_list_key, + test_manifest_list_avro_entries(&[(&manifest, 1, 10), (&second_manifest, 2, 11)]), + ) + .await; + seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; + let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": second_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + } + ] + })) + .expect("second append request should parse"); + + let upgraded = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) + .await + .expect("manifest-list snapshot should inherit a legacy manifest with unknown provenance"); + + assert_eq!(upgraded.metadata["current-snapshot-id"], 11); + assert_eq!(upgraded.metadata["last-sequence-number"], 2); +} + +#[tokio::test] +async fn row_level_conflict_inherits_manifest_list_sequence_numbers() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let manifest = format!("{table_location}/metadata/manifest-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 10).await; + seed_test_manifest_with_nullable_sequences(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, None)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect("manifest entry should inherit sequence numbers from manifest list"); + + assert_eq!(commit.metadata["current-snapshot-id"], 10); + assert_eq!(commit.metadata["last-sequence-number"], 1); +} + +#[tokio::test] +async fn row_level_conflict_allows_inherited_manifests_on_append() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let first_manifest = format!("{table_location}/metadata/manifest-10.avro"); + let first_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; + seed_test_manifest(&metadata_backend, "warehouse", &first_manifest, &[(&first_data_file, 0, 1, 10, 1)]).await; + let first_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": first_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("first append request should parse"); + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_append) + .await + .expect("first append should commit"); + + let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); + let second_data_file = format!("{table_location}/data/part-11.parquet"); + let second_manifest_list_key = test_snapshot_object_key("warehouse", &second_manifest_list); + metadata_backend + .put_bytes( + "warehouse", + &second_manifest_list_key, + test_manifest_list_avro_entries(&[(&first_manifest, 1, 10), (&second_manifest, 2, 11)]), + ) + .await; + seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; + let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": second_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + } + ] + })) + .expect("second append request should parse"); + + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) + .await + .expect("append should preserve inherited manifests"); + + assert_eq!(commit.metadata["current-snapshot-id"], 11); + assert_eq!(commit.metadata["last-sequence-number"], 2); +} + +#[tokio::test] +async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let first_manifest = format!("{table_location}/metadata/manifest-10.avro"); + let first_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; + seed_test_manifest(&metadata_backend, "warehouse", &first_manifest, &[(&first_data_file, 0, 1, 10, 1)]).await; + let first_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": first_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("first append request should parse"); + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_append) + .await + .expect("first append should commit"); + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + + let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); + seed_test_manifest_list(&metadata_backend, "warehouse", &second_manifest_list, &[&first_manifest], 2, 10).await; + let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": second_manifest_list, + "summary": { + "operation": "append" + } + } + } + ] + })) + .expect("second append request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) + .await + .expect_err("inherited manifest identity must not change"); + + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current.metadata_location); + assert_eq!(unchanged.version_token, current.version_token); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn row_level_conflict_rejects_stale_new_manifest_sequence() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let manifest = format!("{table_location}/metadata/manifest-11.avro"); + let data_file = format!("{table_location}/data/part-11.parquet"); + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 11).await; + seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 11, 1)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + } + ] + })) + .expect("append request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect_err("new manifest sequence must match the committed snapshot"); + + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current.metadata_location); + assert_eq!(unchanged.version_token, current.version_token); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn row_level_conflict_rejects_stale_added_entry_sequence() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let manifest = format!("{table_location}/metadata/manifest-11.avro"); + let data_file = format!("{table_location}/data/part-11.parquet"); + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 2, 11).await; + seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 11, 1)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + } + ] + })) + .expect("append request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect_err("added file sequence must match the new manifest"); + + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current.metadata_location); + assert_eq!(unchanged.version_token, current.version_token); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn row_level_conflict_rejects_historical_change_in_new_manifest() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let manifest = format!("{table_location}/metadata/manifest-11.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 2, 11).await; + seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, 1)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + } + ] + })) + .expect("append request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect_err("new manifest must not claim a historical changed entry"); + + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current.metadata_location); + assert_eq!(unchanged.version_token, current.version_token); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn row_level_conflict_allows_add_only_overwrite_snapshot() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let current_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + ¤t_manifest_list, + 10, + 1, + &[(¤t_data_file, 0, 1, 10, 1)], + ) + .await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": current_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect("append commit should succeed"); + + let overwrite_manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let added_data_file = format!("{table_location}/data/part-11.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + &overwrite_manifest_list, + 11, + 2, + &[(&added_data_file, 0, 1, 11, 2)], + ) + .await; + let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": overwrite_manifest_list, + "summary": { + "operation": "overwrite" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + } + ] + })) + .expect("overwrite request should parse"); + + let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) + .await + .expect("add-only overwrite should pass conflict validation"); + + assert_eq!(commit.metadata["current-snapshot-id"], 11); + assert_eq!(commit.metadata["last-sequence-number"], 2); +} + +#[tokio::test] +async fn row_level_conflict_rejects_delete_of_non_current_file() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let current_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + ¤t_manifest_list, + 10, + 1, + &[(¤t_data_file, 0, 1, 10, 1)], + ) + .await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": current_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect("append commit should succeed"); + let committed = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + + let stale_data_file = format!("{table_location}/data/stale.parquet"); + let stale_key = test_snapshot_object_key("warehouse", &stale_data_file); + metadata_backend.put_bytes("warehouse", &stale_key, b"stale".to_vec()).await; + let overwrite_manifest_list = format!("{table_location}/metadata/snap-11.avro"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + &overwrite_manifest_list, + 11, + 2, + &[(&stale_data_file, 0, 2, 11, 2)], + ) + .await; + let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": overwrite_manifest_list, + "summary": { + "operation": "overwrite" + } + } + } + ] + })) + .expect("overwrite request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) + .await + .expect_err("stale row-level delete should conflict"); + + assert_eq!(error.code(), &s3s::S3ErrorCode::PreconditionFailed); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, committed.metadata_location); + assert_eq!(unchanged.version_token, committed.version_token); + assert_eq!(unchanged.generation, committed.generation); +} + +#[tokio::test] +async fn row_level_conflict_rejects_append_with_delete_files() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let delete_file = format!("{table_location}/delete/delete-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&delete_file, 1, 1, 10, 1)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + } + ] + })) + .expect("append request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect_err("append must not add delete files"); + + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current.metadata_location); + assert_eq!(unchanged.version_token, current.version_token); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn row_level_conflict_rejects_missing_manifest_before_pointer_update() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let current_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + ¤t_manifest_list, + 10, + 1, + &[(¤t_data_file, 0, 1, 10, 1)], + ) + .await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": current_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect("append commit should succeed"); + let committed = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let missing_manifest_list = format!("{table_location}/metadata/missing-snap-11.avro"); + let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": missing_manifest_list, + "summary": { + "operation": "overwrite" + } + } + } + ] + })) + .expect("overwrite request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) + .await + .expect_err("missing manifest-list should fail before pointer update"); + + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, committed.metadata_location); + assert_eq!(unchanged.version_token, committed.version_token); + assert_eq!(unchanged.generation, committed.generation); +} + +#[tokio::test] +async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let current_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + ¤t_manifest_list, + 10, + 1, + &[(¤t_data_file, 0, 1, 10, 1)], + ) + .await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": current_manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect("append commit should succeed"); + let committed = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let outside_manifest_list = "s3://warehouse/tables/other-table/metadata/snap-11.avro"; + let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": outside_manifest_list, + "summary": { + "operation": "overwrite" + } + } + } + ] + })) + .expect("overwrite request should parse"); + + let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) + .await + .expect_err("outside manifest-list should fail before pointer update"); + + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, committed.metadata_location); + assert_eq!(unchanged.version_token, committed.version_token); + assert_eq!(unchanged.generation, committed.generation); +} + +#[tokio::test] +async fn bodyless_ref_delete_uses_default_request_options() { + let request: DeleteTableRefRequest = read_json_body_or_default(Body::empty()) + .await + .expect("bodyless ref delete should use default request options"); + + assert!(request.expected_snapshot_id.is_none()); + assert!(!request.force); + assert!(request.commit_id.is_none()); + assert!(request.idempotency_key.is_none()); + assert!(request.writer.is_none()); +} + +#[test] +fn table_updates_reject_unknown_actions() { + let metadata = serde_json::json!({ + "metadata-log": [] + }); + let updates = vec![serde_json::json!({ + "action": "rewrite-everything" + })]; + + assert!(apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json").is_err()); +} + +#[test] +fn table_location_updates_must_stay_inside_bucket() { + let metadata = serde_json::json!({ + "location": "s3://warehouse/tables/table-id", + "metadata-log": [] + }); + let updates = vec![serde_json::json!({ + "action": "set-location", + "location": "s3://other-warehouse/tables/table-id" + })]; + + let updated = apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json") + .expect("set-location should update metadata before boundary validation"); + + assert!(validate_metadata_table_location_in_bucket("warehouse", &updated).is_err()); +} + +#[test] +fn create_view_request_accepts_standard_iceberg_rest_shape() { + let request: CreateViewRequest = serde_json::from_value(serde_json::json!({ + "name": "recent_events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + } + ] + }, + "view-version": { + "version-id": 1, + "schema-id": 0, + "summary": { + "engine-name": "spark", + "engine-version": "3.5.0" + }, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [ + { + "type": "sql", + "sql": "SELECT id FROM analytics.events WHERE ts >= current_date()", + "dialect": "spark" + } + ] + }, + "properties": { + "comment": "recent event ids" + } + })) + .expect("standard create view request should parse"); + + assert_eq!(request.name, "recent_events"); + assert_eq!(request.properties.get("comment").map(String::as_str), Some("recent event ids")); +} + +#[test] +fn create_view_request_accepts_deep_warehouse_location() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let deep_prefix = (0..80).map(|index| format!("level-{index}")).collect::>().join("/"); + let location = format!("s3://warehouse/{deep_prefix}"); + assert!(crate::table_catalog::validate_table_warehouse_location("warehouse", &location).is_err()); + let request: CreateViewRequest = serde_json::from_value(serde_json::json!({ + "name": "recent_events", + "location": location, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [] + }, + "view-version": { + "version-id": 1, + "schema-id": 0, + "summary": { + "engine-name": "spark" + }, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [ + { + "type": "sql", + "sql": "SELECT 1", + "dialect": "spark" + } + ] + } + })) + .expect("deep create view request should parse"); + + let (entry, metadata) = view_entry_from_create_view_request("warehouse", &namespace, request) + .expect("view location should not inherit table warehouse index depth limits"); + + assert_eq!(entry.warehouse_location, format!("s3://warehouse/{deep_prefix}")); + validate_metadata_view_location_in_bucket("warehouse", &metadata) + .expect("view metadata location should not inherit table warehouse index depth limits"); +} + +#[tokio::test] +async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + + let create_request: CreateViewRequest = serde_json::from_value(serde_json::json!({ + "name": "recent_events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + } + ] + }, + "view-version": { + "version-id": 1, + "schema-id": 0, + "summary": { + "engine-name": "spark" + }, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [ + { + "type": "sql", + "sql": "SELECT id FROM analytics.events", + "dialect": "spark" + } + ] + } + })) + .expect("standard create view request should parse"); + + let created = create_view_response(&store, &metadata_backend, "warehouse", &namespace, create_request, true) + .await + .expect("view should be created"); + assert_eq!(created.metadata["format-version"], 1); + assert_eq!(created.metadata["current-version-id"], 1); + assert_eq!(created.metadata["versions"][0]["representations"][0]["dialect"], "spark"); + assert!( + metadata_backend + .object_exists("warehouse", &created.metadata_location) + .await + .expect("view metadata object lookup should succeed") + ); + + let unpaginated_uri = "/".parse::().expect("list URI should parse"); + let listed = list_views_response(&store, "warehouse", &namespace, &unpaginated_uri) + .await + .expect("views should list"); + assert_eq!(listed.identifiers.len(), 1); + assert_eq!(listed.identifiers[0].name, "recent_events"); + + let loaded = load_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events") + .await + .expect("view should load"); + assert_eq!(loaded.metadata_location, created.metadata_location); + let replace_request: RestCommitViewRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-view-version", + "view-version": { + "version-id": 2, + "schema-id": 0, + "summary": { + "engine-name": "spark" + }, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [ + { + "type": "sql", + "sql": "SELECT id FROM analytics.events WHERE id > 10", + "dialect": "spark" + } + ] + } + }, + { + "action": "set-current-view-version", + "view-version-id": 2 + } + ] + })) + .expect("replace view request should parse"); + let replaced = replace_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events", replace_request) + .await + .expect("view should replace"); + assert_ne!(replaced.metadata_location, created.metadata_location); + assert_eq!(replaced.metadata["current-version-id"], 2); + assert_eq!( + replaced.metadata["version-log"] + .as_array() + .expect("version log should be an array") + .len(), + 2 + ); + + drop_view_in_store(&store, "warehouse", &namespace, "recent_events") + .await + .expect("view should drop"); + let listed = list_views_response(&store, "warehouse", &namespace, &unpaginated_uri) + .await + .expect("views should list after drop"); + assert!(listed.identifiers.is_empty()); + + let recreate_request: CreateViewRequest = serde_json::from_value(serde_json::json!({ + "name": "recent_events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + } + ] + }, + "view-version": { + "version-id": 1, + "schema-id": 0, + "summary": { + "engine-name": "spark" + }, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [ + { + "type": "sql", + "sql": "SELECT id FROM analytics.events WHERE id > 100", + "dialect": "spark" + } + ] + } + })) + .expect("standard recreate view request should parse"); + let recreated = create_view_response(&store, &metadata_backend, "warehouse", &namespace, recreate_request, true) + .await + .expect("dropped view name should be reusable"); + assert_ne!(recreated.metadata_location, created.metadata_location); +} + +#[tokio::test] +async fn table_ref_write_responses_commit_retention_refs_and_protect_deletes() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) + .await + .expect("append should commit"); + + let ref_request: PutTableRefRequest = serde_json::from_value(serde_json::json!({ + "snapshot-id": 10, + "type": "tag", + "max-ref-age-ms": 86400000, + "expected-snapshot-id": null + })) + .expect("ref put request should parse"); + put_table_ref_response(&store, &metadata_backend, "warehouse", &namespace, "events", "audit", ref_request) + .await + .expect("ref put should commit"); + + let refs = table_refs_response(&store, &metadata_backend, "warehouse", &namespace, "events") + .await + .expect("refs should load"); + assert_eq!(refs.refs["audit"]["type"], "tag"); + assert_eq!(refs.refs["audit"]["max-ref-age-ms"], 86400000); + + let delete_without_force: DeleteTableRefRequest = + serde_json::from_value(serde_json::json!({})).expect("ref delete request should parse"); + let error = delete_table_ref_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "events", + "audit", + delete_without_force, + ) + .await + .expect_err("retention refs should require force delete"); + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + + let force_delete: DeleteTableRefRequest = + serde_json::from_value(serde_json::json!({ "force": true })).expect("ref force delete should parse"); + delete_table_ref_response(&store, &metadata_backend, "warehouse", &namespace, "events", "audit", force_delete) + .await + .expect("force delete should commit"); + let refs = table_refs_response(&store, &metadata_backend, "warehouse", &namespace, "events") + .await + .expect("refs should load after delete"); + assert!(!refs.refs.contains_key("audit")); + + let main_delete: DeleteTableRefRequest = + serde_json::from_value(serde_json::json!({ "force": true })).expect("main delete request should parse"); + let error = delete_table_ref_response(&store, &metadata_backend, "warehouse", &namespace, "events", "main", main_delete) + .await + .expect_err("main ref should remain protected"); + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); +} + +#[test] +fn load_table_response_includes_rest_metadata_payload() { + let metadata = serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "metadata-log": [ + { + "timestamp-ms": 1, + "metadata-file": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json" + }, + { + "timestamp-ms": 2, + "metadata-file": "s3://warehouse/external/metadata/00000.metadata.json" + } + ] + }); + let response = load_table_response_from_entry(table_entry_for_credentials(), metadata); + + assert_eq!( + response.metadata_location, + "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json" + ); + assert_eq!( + response.metadata["metadata-log"][0]["metadata-file"], + "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json" + ); + assert_eq!( + response.metadata["metadata-log"][1]["metadata-file"], + "s3://warehouse/external/metadata/00000.metadata.json" + ); + assert!(response.storage_credentials.is_empty()); + assert_eq!(response.config.get("rustfs.credential-vending"), Some(&"unsupported".to_string())); + assert_eq!( + response.config.get("rustfs.credential-vending-reason"), + Some(&"temporary-credentials-not-implemented".to_string()) + ); + assert_eq!(response.config.get("rustfs.credential-scope"), Some(&"table-prefix".to_string())); + assert_eq!( + response.config.get("rustfs.credential-scope-prefix"), + Some(&"s3://warehouse/tables/table-id".to_string()) + ); + assert_eq!( + response.config.get("rustfs.credential-mode"), + Some(&"client-provided-s3-credentials-required".to_string()) + ); + assert!(!response.config.contains_key("s3.access-key-id")); + assert!(!response.config.contains_key("s3.secret-access-key")); + assert!(!response.config.contains_key("s3.session-token")); +} + +#[test] +fn load_table_response_preserves_format_v4_relative_metadata_log() { + let metadata = serde_json::json!({ + "format-version": 4, + "table-uuid": "table-uuid", + "metadata-log": [ + { + "timestamp-ms": 1, + "metadata-file": "metadata/00000.metadata.json" + }, + { + "timestamp-ms": 2, + "metadata-file": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json" + } + ] + }); + + let response = load_table_response_from_entry(table_entry_for_credentials(), metadata); + + assert_eq!(response.metadata["metadata-log"][0]["metadata-file"], "metadata/00000.metadata.json"); + assert_eq!( + response.metadata["metadata-log"][1]["metadata-file"], + "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00000.metadata.json" + ); +} + +#[test] +fn table_metadata_location_for_catalog_accepts_only_the_table_bucket() { + let object_key = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + + assert_eq!( + table_metadata_location_for_catalog("warehouse", &format!("s3://warehouse/{object_key}")) + .expect("same-bucket metadata location should normalize"), + object_key + ); + assert!(table_metadata_location_for_catalog("warehouse", &format!("s3://other/{object_key}")).is_err()); +} + +fn table_entry_for_credentials() -> crate::table_catalog::TableEntry { + crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json" + .to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + } +} + +#[tokio::test] +async fn disabled_table_credential_issuer_keeps_credentials_empty() { + let issuer = DisabledTableCredentialIssuer; + let response = load_credentials_response_from_entry(&table_entry_for_credentials(), &issuer, None) + .await + .expect("disabled issuer should build an empty response"); + + assert!(response.storage_credentials.is_empty()); + assert_eq!( + response.config.get(CREDENTIAL_VENDING_CONFIG_KEY), + Some(&CREDENTIAL_VENDING_UNSUPPORTED.to_string()) + ); + assert_eq!( + response.config.get(CREDENTIAL_MODE_CONFIG_KEY), + Some(&CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()) + ); + assert_eq!( + response.config.get(CREDENTIAL_VENDING_REASON_CONFIG_KEY), + Some(&"credential-vending-disabled".to_string()) + ); +} + +#[tokio::test] +async fn disabled_table_credential_issuer_skips_scope_validation() { + let issuer = DisabledTableCredentialIssuer; + let mut entry = table_entry_for_credentials(); + entry.warehouse_location = "s3://warehouse/".to_string(); + + let response = load_credentials_response_from_entry(&entry, &issuer, None) + .await + .expect("disabled issuer should not validate credential scopes"); + + assert!(response.storage_credentials.is_empty()); + assert_eq!( + response.config.get(CREDENTIAL_VENDING_CONFIG_KEY), + Some(&CREDENTIAL_VENDING_UNSUPPORTED.to_string()) + ); + assert!(!response.config.contains_key(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY)); +} + +struct UnavailableTableCredentialIssuer; + +#[async_trait::async_trait] +impl TableCredentialIssuer for UnavailableTableCredentialIssuer { + async fn issue_table_credentials( + &self, + request: TableCredentialIssueRequest<'_>, + ) -> S3Result> { + assert_eq!(request.scope_prefix, "s3://warehouse/tables/table-id/"); + Ok(None) + } +} + +#[tokio::test] +async fn unavailable_table_credential_issuer_reports_fallback_scope() { + let issuer = UnavailableTableCredentialIssuer; + let response = load_credentials_response_from_entry(&table_entry_for_credentials(), &issuer, None) + .await + .expect("unavailable issuer should build a fallback response"); + + assert!(response.storage_credentials.is_empty()); + assert_eq!( + response.config.get(CREDENTIAL_VENDING_CONFIG_KEY), + Some(&CREDENTIAL_VENDING_UNSUPPORTED.to_string()) + ); + assert_eq!( + response.config.get(CREDENTIAL_MODE_CONFIG_KEY), + Some(&CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()) + ); + assert_eq!( + response.config.get(CREDENTIAL_VENDING_REASON_CONFIG_KEY), + Some(&CREDENTIAL_VENDING_UNSUPPORTED_REASON.to_string()) + ); + assert_eq!( + response.config.get(CREDENTIAL_SCOPE_CONFIG_KEY), + Some(&CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()) + ); + assert_eq!( + response.config.get(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY), + Some(&"s3://warehouse/tables/table-id/".to_string()) + ); +} + +struct TestTableCredentialIssuer; + +#[async_trait::async_trait] +impl TableCredentialIssuer for TestTableCredentialIssuer { + async fn issue_table_credentials( + &self, + request: TableCredentialIssueRequest<'_>, + ) -> S3Result> { + assert_eq!(request.entry.table_bucket, "warehouse"); + assert_eq!(request.scope_prefix, "s3://warehouse/tables/table-id/"); + assert_eq!(request.object_prefix, "tables/table-id/"); + Ok(Some(IssuedTableCredentials { + access_key_id: "temporary-access-key".to_string(), + secret_access_key: "temporary-secret-key".to_string(), + session_token: "temporary-session-token".to_string(), + expiration: OffsetDateTime::from_unix_timestamp(1_800_000_000).expect("test timestamp should be valid"), + })) + } +} + +#[tokio::test] +async fn credential_issuer_returns_temporary_scoped_storage_credentials() { + let issuer = TestTableCredentialIssuer; + let principal = rustfs_credentials::Credentials { + access_key: "parent-access-key".to_string(), + secret_key: "parent-secret-key".to_string(), + ..Default::default() + }; + + let response = load_credentials_response_from_entry(&table_entry_for_credentials(), &issuer, Some(&principal)) + .await + .expect("issuer should build a scoped credential response"); + + assert_eq!( + response.config.get(CREDENTIAL_VENDING_CONFIG_KEY), + Some(&CREDENTIAL_VENDING_SUPPORTED.to_string()) + ); + assert_eq!( + response.config.get(CREDENTIAL_MODE_CONFIG_KEY), + Some(&CREDENTIAL_MODE_CATALOG_VENDED.to_string()) + ); + assert!(!response.config.contains_key(S3_ACCESS_KEY_ID_CONFIG_KEY)); + assert!(!response.config.contains_key(S3_SECRET_ACCESS_KEY_CONFIG_KEY)); + assert!(!response.config.contains_key(S3_SESSION_TOKEN_CONFIG_KEY)); + assert_eq!(response.storage_credentials.len(), 1); + let credential = &response.storage_credentials[0]; + assert_eq!(credential.prefix, "s3://warehouse/tables/table-id/"); + assert_eq!(credential.config.get("s3.access-key-id"), Some(&"temporary-access-key".to_string())); + assert_eq!(credential.config.get("s3.secret-access-key"), Some(&"temporary-secret-key".to_string())); + assert_eq!(credential.config.get("s3.session-token"), Some(&"temporary-session-token".to_string())); + assert_eq!( + credential.config.get("rustfs.credential-mode"), + Some(&"catalog-vended-temporary-credentials".to_string()) + ); + assert_eq!( + credential.config.get("rustfs.credential-scope-prefix"), + Some(&"s3://warehouse/tables/table-id/".to_string()) + ); + assert_eq!( + credential.config.get("rustfs.credential-expiration-unix-seconds"), + Some(&"1800000000".to_string()) + ); + assert!(!credential.config.contains_key("rustfs.credential-vending-reason")); +} + +#[tokio::test] +async fn credential_response_serializes_sensitive_config_only_inside_storage_credentials() { + let issuer = TestTableCredentialIssuer; + let response = load_credentials_response_from_entry(&table_entry_for_credentials(), &issuer, None) + .await + .expect("issuer should build a scoped credential response"); + + let value = serde_json::to_value(&response).expect("credential response should serialize"); + + assert_eq!( + value["config"][CREDENTIAL_VENDING_CONFIG_KEY], + serde_json::Value::String(CREDENTIAL_VENDING_SUPPORTED.to_string()) + ); + assert!(value["config"].get(S3_ACCESS_KEY_ID_CONFIG_KEY).is_none()); + assert!(value["config"].get(S3_SECRET_ACCESS_KEY_CONFIG_KEY).is_none()); + assert!(value["config"].get(S3_SESSION_TOKEN_CONFIG_KEY).is_none()); + assert_eq!( + value["storage-credentials"][0]["config"][S3_ACCESS_KEY_ID_CONFIG_KEY], + serde_json::Value::String("temporary-access-key".to_string()) + ); +} + +#[test] +fn table_credentials_do_not_snapshot_parent_groups() { + let principal = rustfs_credentials::Credentials { + access_key: "parent-access-key".to_string(), + groups: Some(vec!["analytics-writers".to_string()]), + ..Default::default() + }; + let mut credential = rustfs_credentials::Credentials::default(); + + bind_table_credential_parent(&mut credential, &principal); + + assert_eq!(credential.parent_user, "parent-access-key"); + assert!(credential.groups.is_none()); +} + +#[tokio::test] +async fn table_credential_session_policy_is_limited_to_table_prefix() { + let policy = table_credential_session_policy(&table_entry_for_credentials(), "tables/table-id/") + .expect("table credential policy should build"); + let groups = None; + let conditions = std::collections::HashMap::new(); + let claims = std::collections::HashMap::new(); + + assert!( + policy + .is_allowed(&rustfs_policy::policy::Args { + account: "temporary-access-key", + groups: &groups, + action: Action::S3Action(rustfs_policy::policy::action::S3Action::GetObjectAction), + bucket: "warehouse", + conditions: &conditions, + is_owner: false, + object: "tables/table-id/data/file.parquet", + claims: &claims, + deny_only: false, + }) + .await + ); + assert!( + policy + .is_allowed(&rustfs_policy::policy::Args { + account: "temporary-access-key", + groups: &groups, + action: Action::S3Action(rustfs_policy::policy::action::S3Action::GetBucketLocationAction), + bucket: "warehouse", + conditions: &conditions, + is_owner: false, + object: "", + claims: &claims, + deny_only: false, + }) + .await + ); + assert!( + policy + .is_allowed(&rustfs_policy::policy::Args { + account: "temporary-access-key", + groups: &groups, + action: Action::AdminAction(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction), + bucket: "warehouse", + conditions: &conditions, + is_owner: false, + object: "namespaces/analytics/tables/events", + claims: &claims, + deny_only: false, + }) + .await + ); + assert!( + !policy + .is_allowed(&rustfs_policy::policy::Args { + account: "temporary-access-key", + groups: &groups, + action: Action::S3Action(rustfs_policy::policy::action::S3Action::GetObjectAction), + bucket: "warehouse", + conditions: &conditions, + is_owner: false, + object: "tables/other/data/file.parquet", + claims: &claims, + deny_only: false, + }) + .await + ); + assert!( + !policy + .is_allowed(&rustfs_policy::policy::Args { + account: "temporary-access-key", + groups: &groups, + action: Action::S3Action(rustfs_policy::policy::action::S3Action::PutObjectAction), + bucket: "other-warehouse", + conditions: &conditions, + is_owner: false, + object: "tables/table-id/data/file.parquet", + claims: &claims, + deny_only: false, + }) + .await + ); +} + +#[tokio::test] +async fn table_credential_session_policy_includes_table_resource_actions() { + let policy = table_credential_session_policy(&table_entry_for_credentials(), "tables/table-id/") + .expect("table credential policy should build"); + let groups = None; + let conditions = std::collections::HashMap::new(); + let claims = std::collections::HashMap::new(); + + assert!( + policy + .is_allowed(&rustfs_policy::policy::Args { + account: "temporary-access-key", + groups: &groups, + action: Action::AdminAction(rustfs_policy::policy::action::AdminAction::GetTableMetadataAction), + bucket: "warehouse", + conditions: &conditions, + is_owner: false, + object: "namespaces/analytics/tables/events", + claims: &claims, + deny_only: false, + }) + .await + ); + assert!( + !policy + .is_allowed(&rustfs_policy::policy::Args { + account: "temporary-access-key", + groups: &groups, + action: Action::AdminAction(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction), + bucket: "warehouse", + conditions: &conditions, + is_owner: false, + object: "namespaces/analytics/tables/orders", + claims: &claims, + deny_only: false, + }) + .await + ); + assert!( + !policy + .is_allowed(&rustfs_policy::policy::Args { + account: "temporary-access-key", + groups: &groups, + action: Action::AdminAction(rustfs_policy::policy::action::AdminAction::CreateTableAction), + bucket: "warehouse", + conditions: &conditions, + is_owner: false, + object: "namespaces/analytics/tables/events", + claims: &claims, + deny_only: false, + }) + .await + ); +} + +#[test] +fn table_credential_scope_rejects_cross_bucket_or_unsafe_prefix() { + let mut entry = table_entry_for_credentials(); + entry.warehouse_location = "s3://other-warehouse/tables/table-id".to_string(); + assert!(table_credential_scope(&entry).is_err()); + + let mut entry = table_entry_for_credentials(); + entry.warehouse_location = "s3://warehouse/tables/../table-id".to_string(); + assert!(table_credential_scope(&entry).is_err()); +} + +#[test] +fn commit_table_request_uses_rest_commit_fields() { + let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": "commit-1", + "idempotency-key": "retry-1", + "operation": "append", + "expected-version-token": "token-v1", + "expected-metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json", + "new-metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json", + "requirements": [ + { + "type": "assert-current-snapshot-id", + "snapshot-id": 10 + } + ], + "writer": "pyiceberg" + })) + .expect("commit request should parse"); + + assert_eq!(request.commit_id.as_deref(), Some("commit-1")); + assert_eq!(request.idempotency_key.as_deref(), Some("retry-1")); + assert_eq!(request.operation.as_deref(), Some("append")); + assert_eq!(request.expected_version_token.as_deref(), Some("token-v1")); + assert_eq!( + request.new_metadata_location.as_deref(), + Some(".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json") + ); + assert_eq!(request.requirements.len(), 1); + assert_eq!(request.writer.as_deref(), Some("pyiceberg")); +} + +#[derive(Default)] +struct TestTableCatalogStore { + table_buckets: tokio::sync::Mutex>, + namespaces: tokio::sync::Mutex>, + tables: tokio::sync::Mutex>, + views: tokio::sync::Mutex>, + commits: tokio::sync::Mutex>, + fail_put_table_bucket: tokio::sync::Mutex, +} + +#[derive(Clone, Default)] +struct TestTableCatalogObjectBackend { + objects: Arc>>, + put_object_barrier: Option>, +} + +impl TestTableCatalogObjectBackend { + async fn put_bytes(&self, bucket: &str, object: &str, data: Vec) { + self.objects.lock().await.insert( + (bucket.to_string(), object.to_string()), + crate::table_catalog::TableCatalogObject { + data, + etag: Some("etag".to_string()), + mod_time: None, + }, + ); + } + + async fn put_json(&self, bucket: &str, object: &str, value: serde_json::Value) { + self.put_json_with_mod_time(bucket, object, value, None).await; + } + + async fn put_json_with_mod_time( + &self, + bucket: &str, + object: &str, + value: serde_json::Value, + mod_time: Option, + ) { + let data = serde_json::to_vec(&value).expect("metadata JSON should serialize"); + self.objects.lock().await.insert( + (bucket.to_string(), object.to_string()), + crate::table_catalog::TableCatalogObject { + data, + etag: Some("etag".to_string()), + mod_time, + }, + ); + } +} + +fn test_snapshot_object_key(bucket: &str, location: &str) -> String { + crate::table_catalog::table_catalog_object_key_from_location(bucket, location) + .expect("test snapshot object location should be valid") +} + +fn test_manifest_list_avro_bytes(manifest_paths: &[&str], sequence_number: i64, snapshot_id: i64) -> Vec { + let manifests = manifest_paths + .iter() + .map(|manifest_path| (*manifest_path, sequence_number, snapshot_id)) + .collect::>(); + test_manifest_list_avro_entries(&manifests) +} + +fn test_manifest_list_avro_entries(manifests: &[(&str, i64, i64)]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string"}, + {"name": "sequence_number", "type": "long"}, + {"name": "added_snapshot_id", "type": "long"} + ] + } + "#, + ) + .expect("manifest list avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + for (manifest_path, sequence_number, snapshot_id) in manifests { + writer + .append(apache_avro::types::Value::Record(vec![ + ( + "manifest_path".to_string(), + apache_avro::types::Value::String((*manifest_path).to_string()), + ), + ("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), + ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), + ])) + .expect("manifest list record should append"); + } + writer.into_inner().expect("manifest list avro bytes should flush") +} + +fn test_manifest_avro_bytes(files: &[(&str, i32, i32, i64, i64)]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": "long"}, + {"name": "sequence_number", "type": "long"}, + {"name": "file_sequence_number", "type": "long"}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "data_file", + "fields": [ + {"name": "content", "type": "int"}, + {"name": "file_path", "type": "string"}, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"} + ] + } + } + ] + } + "#, + ) + .expect("manifest avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + for (file_path, content, status, snapshot_id, sequence_number) in files { + writer + .append(apache_avro::types::Value::Record(vec![ + ("status".to_string(), apache_avro::types::Value::Int(*status)), + ("snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), + ("file_sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), + ( + "data_file".to_string(), + apache_avro::types::Value::Record(vec![ + ("content".to_string(), apache_avro::types::Value::Int(*content)), + ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), + ("record_count".to_string(), apache_avro::types::Value::Long(1)), + ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), + ]), + ), + ])) + .expect("manifest record should append"); + } + writer.into_inner().expect("manifest avro bytes should flush") +} + +fn test_nullable_long(value: Option) -> apache_avro::types::Value { + match value { + Some(value) => apache_avro::types::Value::Union(1, Box::new(apache_avro::types::Value::Long(value))), + None => apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + } +} + +fn test_manifest_avro_bytes_with_nullable_sequences(files: &[(&str, i32, i32, i64, Option)]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": "long"}, + {"name": "sequence_number", "type": ["null", "long"], "default": null}, + {"name": "file_sequence_number", "type": ["null", "long"], "default": null}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "data_file", + "fields": [ + {"name": "content", "type": "int"}, + {"name": "file_path", "type": "string"}, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"} + ] + } + } + ] + } + "#, + ) + .expect("manifest avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + for (file_path, content, status, snapshot_id, sequence_number) in files { + writer + .append(apache_avro::types::Value::Record(vec![ + ("status".to_string(), apache_avro::types::Value::Int(*status)), + ("snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), + ("sequence_number".to_string(), test_nullable_long(*sequence_number)), + ("file_sequence_number".to_string(), test_nullable_long(*sequence_number)), + ( + "data_file".to_string(), + apache_avro::types::Value::Record(vec![ + ("content".to_string(), apache_avro::types::Value::Int(*content)), + ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), + ("record_count".to_string(), apache_avro::types::Value::Long(1)), + ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), + ]), + ), + ])) + .expect("manifest record should append"); + } + writer.into_inner().expect("manifest avro bytes should flush") +} + +async fn seed_test_manifest_list( + backend: &TestTableCatalogObjectBackend, + bucket: &str, + manifest_list_location: &str, + manifest_locations: &[&str], + sequence_number: i64, + snapshot_id: i64, +) { + let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); + backend + .put_bytes( + bucket, + &manifest_list_key, + test_manifest_list_avro_bytes(manifest_locations, sequence_number, snapshot_id), + ) + .await; +} + +async fn seed_test_snapshot_manifest( + backend: &TestTableCatalogObjectBackend, + bucket: &str, + manifest_list_location: &str, + snapshot_id: i64, + sequence_number: i64, + files: &[(&str, i32, i32, i64, i64)], +) { + let manifest_location = manifest_list_location + .rsplit_once('/') + .map(|(prefix, name)| format!("{prefix}/manifest-{name}")) + .expect("manifest list location should include a file name"); + let manifest_key = test_snapshot_object_key(bucket, &manifest_location); + let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); + backend + .put_bytes( + bucket, + &manifest_list_key, + test_manifest_list_avro_bytes(&[&manifest_location], sequence_number, snapshot_id), + ) + .await; + backend + .put_bytes(bucket, &manifest_key, test_manifest_avro_bytes(files)) + .await; + seed_test_manifest_data_files(backend, bucket, files).await; +} + +async fn seed_test_manifest_with_nullable_sequences( + backend: &TestTableCatalogObjectBackend, + bucket: &str, + manifest_location: &str, + files: &[(&str, i32, i32, i64, Option)], +) { + let manifest_key = test_snapshot_object_key(bucket, manifest_location); + backend + .put_bytes(bucket, &manifest_key, test_manifest_avro_bytes_with_nullable_sequences(files)) + .await; + let data_files = files + .iter() + .map(|(file_path, content, status, snapshot_id, sequence_number)| { + (*file_path, *content, *status, *snapshot_id, sequence_number.unwrap_or_default()) + }) + .collect::>(); + seed_test_manifest_data_files(backend, bucket, &data_files).await; +} + +async fn seed_test_manifest( + backend: &TestTableCatalogObjectBackend, + bucket: &str, + manifest_location: &str, + files: &[(&str, i32, i32, i64, i64)], +) { + let manifest_key = test_snapshot_object_key(bucket, manifest_location); + backend + .put_bytes(bucket, &manifest_key, test_manifest_avro_bytes(files)) + .await; + seed_test_manifest_data_files(backend, bucket, files).await; +} + +async fn seed_test_manifest_data_files( + backend: &TestTableCatalogObjectBackend, + bucket: &str, + files: &[(&str, i32, i32, i64, i64)], +) { + for (file_path, _, status, _, _) in files { + if *status != 2 { + let object_key = test_snapshot_object_key(bucket, file_path); + backend.put_bytes(bucket, &object_key, b"data".to_vec()).await; + } + } +} + +async fn create_standard_events_table( + store: &TestTableCatalogStore, + metadata_backend: &TestTableCatalogObjectBackend, + namespace: &crate::table_catalog::Namespace, +) -> RestLoadTableResponse { + ensure_table_bucket_entry(store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + store, + "warehouse", + CreateNamespaceRequest { + namespace: namespace.public_name().split('.').map(str::to_string).collect(), + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let create_request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + } + ] + } + })) + .expect("standard create table request should parse"); + create_table_response(store, metadata_backend, "warehouse", namespace, create_request, true) + .await + .expect("table should be created") +} + +async fn seed_object_table_for_metadata_maintenance( + store: &crate::table_catalog::ObjectTableCatalogStore, + backend: &TestTableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &crate::table_catalog::IdentifierSegment, + current_metadata_location: String, +) { + store + .put_table_bucket(crate::table_catalog::TableBucketEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + catalog_type: crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(), + warehouse_root: format!("s3://{bucket}/"), + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) + .await + .expect("table bucket entry should seed"); + store + .create_namespace(crate::table_catalog::NamespaceEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + namespace_id: namespace.storage_id(), + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) + .await + .expect("namespace entry should seed"); + store + .create_table(crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: format!("s3://{bucket}/tables/table-id"), + metadata_location: current_metadata_location, + version_token: "token-v1".to_string(), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) + .await + .expect("table entry should seed"); + backend + .put_json(bucket, "unrelated/ignored.json", serde_json::json!({})) + .await; +} + +#[async_trait::async_trait] +impl crate::table_catalog::TableCatalogObjectBackend for TestTableCatalogObjectBackend { + async fn read_object( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .objects + .lock() + .await + .get(&(bucket.to_string(), object.to_string())) + .cloned()) + } + + async fn object_exists(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult { + Ok(self + .objects + .lock() + .await + .contains_key(&(bucket.to_string(), object.to_string()))) + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: crate::table_catalog::TableCatalogPutPrecondition, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + let key = (bucket.to_string(), object.to_string()); + let mut objects = self.objects.lock().await; + let result = if matches!(precondition, crate::table_catalog::TableCatalogPutPrecondition::IfAbsent) + && objects.contains_key(&key) + { + Err(crate::table_catalog::TableCatalogStoreError::Conflict(format!( + "object already exists: {object}" + ))) + } else { + objects.insert( + key, + crate::table_catalog::TableCatalogObject { + data, + etag: Some("etag".to_string()), + mod_time: None, + }, + ); + Ok(()) + }; + drop(objects); + if let Some(barrier) = &self.put_object_barrier { + barrier.wait().await; + } + result + } + + async fn delete_object(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.objects.lock().await.remove(&(bucket.to_string(), object.to_string())); + Ok(()) + } + + async fn list_objects(&self, bucket: &str, prefix: &str) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .objects + .lock() + .await + .keys() + .filter(|(object_bucket, object)| object_bucket == bucket && object.starts_with(prefix)) + .map(|(_, object)| object.clone()) + .collect()) + } + + async fn acquire_write_lock( + &self, + _bucket: &str, + _object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(Box::new(())) + } +} + +#[async_trait::async_trait] +impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { + async fn get_table_bucket( + &self, + table_bucket: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .table_buckets + .lock() + .await + .iter() + .find(|entry| entry.table_bucket == table_bucket) + .cloned()) + } + + async fn put_table_bucket( + &self, + entry: crate::table_catalog::TableBucketEntry, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + let mut fail_put_table_bucket = self.fail_put_table_bucket.lock().await; + if *fail_put_table_bucket { + *fail_put_table_bucket = false; + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "injected table bucket write failure".to_string(), + )); + } + drop(fail_put_table_bucket); + + let mut table_buckets = self.table_buckets.lock().await; + table_buckets.retain(|existing| existing.table_bucket != entry.table_bucket); + table_buckets.push(entry); + Ok(()) + } + + async fn create_namespace( + &self, + entry: crate::table_catalog::NamespaceEntry, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + if self.get_table_bucket(&entry.table_bucket).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table bucket {}", + entry.table_bucket + ))); + } + self.namespaces.lock().await.push(entry); + Ok(()) + } + + async fn list_namespaces( + &self, + table_bucket: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .namespaces + .lock() + .await + .iter() + .filter(|entry| entry.table_bucket == table_bucket) + .cloned() + .collect()) + } + + async fn get_namespace( + &self, + table_bucket: &str, + namespace: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .namespaces + .lock() + .await + .iter() + .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) + .cloned()) + } + + async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.namespaces + .lock() + .await + .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace)); + Ok(()) + } + + async fn create_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { + if self.get_table_bucket(&entry.table_bucket).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table bucket {}", + entry.table_bucket + ))); + } + if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + self.tables.lock().await.push(entry); + Ok(()) + } + + async fn register_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { + if self.get_table_bucket(&entry.table_bucket).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table bucket {}", + entry.table_bucket + ))); + } + if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + self.tables.lock().await.push(entry); + Ok(()) + } + + async fn list_tables( + &self, + table_bucket: &str, + namespace: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .tables + .lock() + .await + .iter() + .filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) + .cloned() + .collect()) + } + + async fn load_table( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .tables + .lock() + .await + .iter() + .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table) + .cloned()) + } + + async fn commit_table( + &self, + request: crate::table_catalog::TableCommitRequest, + ) -> crate::table_catalog::TableCatalogStoreResult { + let mut tables = self.tables.lock().await; + let Some(index) = tables.iter().position(|entry| { + entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.table == request.table + }) else { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + request.table_bucket, request.namespace, request.table + ))); + }; + + let current = tables[index].clone(); + if current.version_token != request.expected_version_token { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict( + "current table version token does not match expected token".to_string(), + )); + } + if current.metadata_location != request.expected_metadata_location { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict( + "current table metadata location does not match expected location".to_string(), + )); + } + + let mut next = current.clone(); + next.metadata_location = request.new_metadata_location.clone(); + next.version_token = "token-committed".to_string(); + next.generation = next.generation.saturating_add(1); + tables[index] = next.clone(); + drop(tables); + + let commit_log = crate::table_catalog::CommitLogEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + commit_id: request.commit_id, + idempotency_key: request.idempotency_key, + table_id: current.table_id, + operation: request.operation, + expected_version_token: request.expected_version_token, + new_version_token: next.version_token.clone(), + previous_metadata_location: request.expected_metadata_location, + new_metadata_location: request.new_metadata_location, + requirements: request.requirements, + status: crate::table_catalog::CommitLogStatus::Committed, + writer: request.writer, + created_at: None, + updated_at: None, + }; + self.commits.lock().await.push(commit_log.clone()); + + Ok(crate::table_catalog::TableCommitResult { table: next, commit_log }) + } + + async fn drop_table( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.tables + .lock() + .await + .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table)); + Ok(()) + } + + async fn create_view(&self, entry: crate::table_catalog::ViewEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { + if self.get_table_bucket(&entry.table_bucket).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table bucket {}", + entry.table_bucket + ))); + } + if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + self.views.lock().await.push(entry); + Ok(()) + } + + async fn list_views( + &self, + table_bucket: &str, + namespace: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .views + .lock() + .await + .iter() + .filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) + .cloned() + .collect()) + } + + async fn load_view( + &self, + table_bucket: &str, + namespace: &str, + view: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .views + .lock() + .await + .iter() + .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view) + .cloned()) + } + + async fn replace_view( + &self, + request: crate::table_catalog::ViewCommitRequest, + ) -> crate::table_catalog::TableCatalogStoreResult { + let mut views = self.views.lock().await; + let Some(index) = views.iter().position(|entry| { + entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.view == request.view + }) else { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + request.table_bucket, request.namespace, request.view + ))); + }; + let current = views[index].clone(); + if current.version_token != request.expected_version_token { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict( + "current view version token does not match expected token".to_string(), + )); + } + if current.metadata_location != request.expected_metadata_location { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict( + "current view metadata location does not match expected location".to_string(), + )); + } + let mut next = current; + next.metadata_location = request.new_metadata_location; + next.version_token = "token-view-committed".to_string(); + next.generation = next.generation.saturating_add(1); + views[index] = next.clone(); + Ok(crate::table_catalog::ViewCommitResult { view: next }) + } + + async fn drop_view( + &self, + table_bucket: &str, + namespace: &str, + view: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.views + .lock() + .await + .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view)); + Ok(()) + } + + async fn get_commit_by_id( + &self, + _table_bucket: &str, + _table_id: &str, + _commit_id: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(None) + } + + async fn get_commit_by_idempotency_key( + &self, + _table_bucket: &str, + _table_id: &str, + _idempotency_key: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(None) + } +} + +#[tokio::test] +async fn ensure_table_bucket_entry_seeds_enabled_bucket_before_namespace_create() { + let store = TestTableCatalogStore::default(); + + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + let table_bucket = store + .get_table_bucket("warehouse") + .await + .expect("table bucket lookup should succeed") + .expect("table bucket entry should exist"); + + assert_eq!(table_bucket.table_bucket, "warehouse"); + assert_eq!(table_bucket.catalog_type, crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE); + assert_eq!(table_bucket.warehouse_root, "s3://warehouse/"); +} + +#[tokio::test] +async fn ensure_table_bucket_entry_rejects_bucket_without_table_marker() { + let store = TestTableCatalogStore::default(); + + assert!(ensure_table_bucket_entry(&store, "warehouse", false).await.is_err()); + assert!( + store + .get_table_bucket("warehouse") + .await + .expect("table bucket lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn ensure_table_bucket_entry_propagates_catalog_entry_failure() { + let store = TestTableCatalogStore::default(); + *store.fail_put_table_bucket.lock().await = true; + + assert!(ensure_table_bucket_entry(&store, "warehouse", true).await.is_err()); + assert!( + store + .get_table_bucket("warehouse") + .await + .expect("table bucket lookup should succeed") + .is_none() + ); + assert!(!*store.fail_put_table_bucket.lock().await); +} + +#[tokio::test] +async fn namespace_helpers_call_catalog_store() { + let store = TestTableCatalogStore::default(); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + let create = create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), + }, + true, + ) + .await + .expect("namespace should be created"); + + assert_eq!(create.namespace, vec!["analytics".to_string()]); + assert_eq!(create.properties.get("owner").map(String::as_str), Some("lakehouse")); + + let unpaginated_uri = "/".parse::().expect("list URI should parse"); + let list = list_namespaces_response(&store, "warehouse", &unpaginated_uri) + .await + .expect("namespace list should load"); + assert_eq!(list.namespaces, vec![vec!["analytics".to_string()]]); + + drop_namespace_in_store(&store, "warehouse", "analytics") + .await + .expect("namespace should drop"); + let list = list_namespaces_response(&store, "warehouse", &unpaginated_uri) + .await + .expect("namespace list should load after drop"); + assert!(list.namespaces.is_empty()); +} + +#[tokio::test] +async fn table_helpers_call_catalog_store() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + + let metadata_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + metadata_backend + .put_json( + "warehouse", + metadata_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + let register = register_table_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: metadata_location.to_string(), + overwrite: false, + }, + true, + ) + .await + .expect("table should register"); + + let client_metadata_location = table_metadata_location_for_client("warehouse", metadata_location); + assert_eq!(register.metadata_location, client_metadata_location); + assert_eq!(register.metadata["format-version"], 2); + + let unpaginated_uri = "/".parse::().expect("list URI should parse"); + let list = list_tables_response(&store, "warehouse", &namespace, &unpaginated_uri) + .await + .expect("table list should load"); + assert_eq!(list.identifiers[0].name, "events"); + + let load = load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events") + .await + .expect("table should load"); + assert_eq!(load.metadata_location, client_metadata_location); + assert_eq!(load.metadata["table-uuid"], "table-uuid"); + + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let next_metadata_location = + ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; + metadata_backend + .put_json( + "warehouse", + next_metadata_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-sequence-number": 2 + }), + ) + .await; + let commit = commit_table_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "events", + RestCommitTableRequest { + commit_id: Some("commit-1".to_string()), + idempotency_key: Some("retry-1".to_string()), + operation: Some("append".to_string()), + expected_version_token: Some(current.version_token.clone()), + expected_metadata_location: Some(table_metadata_location_for_client("warehouse", ¤t.metadata_location)), + new_metadata_location: Some(table_metadata_location_for_client("warehouse", next_metadata_location)), + requirements: Vec::new(), + updates: Vec::new(), + _identifier: None, + writer: Some("pyiceberg".to_string()), + }, + ) + .await + .expect("table commit should succeed"); + assert_eq!( + commit.metadata_location, + table_metadata_location_for_client("warehouse", next_metadata_location) + ); + assert_eq!(commit.version_token, "token-committed"); + assert_eq!(commit.generation, current.generation + 1); + assert_eq!(commit.commit_id, "commit-1"); + + let committed = store + .load_table("warehouse", "analytics", "events") + .await + .expect("committed table lookup should succeed") + .expect("committed table should exist"); + assert_eq!(committed.metadata_location, next_metadata_location); + + drop_table_in_store(&store, "warehouse", &namespace, "events") + .await + .expect("table should drop"); + assert!( + load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events") + .await + .is_err() + ); +} + +#[tokio::test] +async fn register_table_response_adopts_metadata_table_uuid() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let metadata_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + metadata_backend + .put_json( + "warehouse", + metadata_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "metadata-table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + register_table_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: metadata_location.to_string(), + overwrite: false, + }, + true, + ) + .await + .expect("table should register"); + + let entry = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + assert_eq!(entry.table_uuid, "metadata-table-uuid"); +} + +#[tokio::test] +async fn register_table_response_rejects_metadata_without_format_version() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let metadata_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + metadata_backend + .put_json( + "warehouse", + metadata_location, + serde_json::json!({ + "table-uuid": "metadata-table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + assert!( + register_table_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: metadata_location.to_string(), + overwrite: false, + }, + true, + ) + .await + .is_err() + ); + assert!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn metadata_location_api_loads_and_updates_current_pointer() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + let entry = table_entry_from_register_request( + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: current_location.to_string(), + overwrite: false, + }, + ) + .expect("table entry should build"); + let table_uuid = entry.table_uuid.clone(); + store.register_table(entry).await.expect("table should register"); + metadata_backend + .put_json( + "warehouse", + current_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": table_uuid, + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") + .await + .expect("metadata location should load"); + assert_eq!( + current.metadata_location, + table_metadata_location_for_client("warehouse", current_location) + ); + let next_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; + metadata_backend + .put_json( + "warehouse", + next_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": table_uuid, + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + let updated = update_table_metadata_location_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "events", + UpdateTableMetadataLocationRequest { + metadata_location: table_metadata_location_for_client("warehouse", next_location), + version_token: current.version_token.clone(), + commit_id: Some("commit-1".to_string()), + idempotency_key: Some("retry-1".to_string()), + }, + ) + .await + .expect("metadata location should update"); + + assert_eq!(updated.metadata_location, table_metadata_location_for_client("warehouse", next_location)); + assert_eq!(updated.generation, current.generation + 1); + assert_ne!(updated.version_token, current.version_token); +} + +#[tokio::test] +async fn metadata_location_api_rejects_invalid_target_metadata_before_commit() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + store + .register_table( + table_entry_from_register_request( + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: current_location.to_string(), + overwrite: false, + }, + ) + .expect("table entry should build"), + ) + .await + .expect("table should register"); + let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") + .await + .expect("metadata location should load"); + let invalid_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; + metadata_backend + .put_json( + "warehouse", + invalid_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://other-warehouse/tables/table-id" + }), + ) + .await; + + assert!( + update_table_metadata_location_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "events", + UpdateTableMetadataLocationRequest { + metadata_location: invalid_location.to_string(), + version_token: current.version_token, + commit_id: Some("commit-1".to_string()), + idempotency_key: None, + }, + ) + .await + .is_err() + ); + let unchanged = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") + .await + .expect("metadata location should still load"); + assert_eq!( + unchanged.metadata_location, + table_metadata_location_for_client("warehouse", current_location) + ); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn metadata_location_api_rejects_mismatched_table_uuid_before_commit() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + metadata_backend + .put_json( + "warehouse", + current_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + register_table_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: current_location.to_string(), + overwrite: false, + }, + true, + ) + .await + .expect("table should register"); + let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") + .await + .expect("metadata location should load"); + let mismatched_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; + metadata_backend + .put_json( + "warehouse", + mismatched_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "other-table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + assert!( + update_table_metadata_location_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "events", + UpdateTableMetadataLocationRequest { + metadata_location: mismatched_location.to_string(), + version_token: current.version_token, + commit_id: Some("commit-1".to_string()), + idempotency_key: None, + }, + ) + .await + .is_err() + ); + let unchanged = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") + .await + .expect("metadata location should still load"); + assert_eq!( + unchanged.metadata_location, + table_metadata_location_for_client("warehouse", current_location) + ); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn catalog_import_and_rollback_use_register_and_commit_paths() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + ensure_table_bucket_entry(&store, bucket, true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + bucket, + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let imported_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + backend + .put_json( + bucket, + &imported_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + let imported = catalog_import_response( + &store, + &backend, + bucket, + &namespace, + "events", + CatalogImportRequest { + metadata_location: table_metadata_location_for_client(bucket, &imported_location), + properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), + }, + true, + ) + .await + .expect("catalog import should register table"); + assert_eq!(imported.metadata_location, table_metadata_location_for_client(bucket, &imported_location)); + let current = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + assert_eq!(current.properties.get("owner").map(String::as_str), Some("lakehouse")); + + let imported_again = catalog_import_response( + &store, + &backend, + bucket, + &namespace, + "events", + CatalogImportRequest { + metadata_location: imported_location.clone(), + properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), + }, + true, + ) + .await + .expect("repeated catalog import should be idempotent"); + assert_eq!( + imported_again.metadata_location, + table_metadata_location_for_client(bucket, &imported_location) + ); + + let rollback_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + backend + .put_json( + bucket, + &rollback_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-sequence-number": 2 + }), + ) + .await; + let rollback = rollback_table_response( + &store, + &backend, + bucket, + &namespace, + "events", + RollbackTableRequest { + metadata_location: table_metadata_location_for_client(bucket, &rollback_location), + version_token: current.version_token, + commit_id: Some("rollback-1".to_string()), + idempotency_key: None, + }, + ) + .await + .expect("rollback should commit selected metadata"); + + assert_eq!(rollback.metadata_location, table_metadata_location_for_client(bucket, &rollback_location)); + assert_eq!(rollback.commit_id, "rollback-1"); +} + +#[tokio::test] +async fn rollback_rejects_invalid_target_metadata_before_commit() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + ensure_table_bucket_entry(&store, bucket, true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + bucket, + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + backend + .put_json( + bucket, + ¤t_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + catalog_import_response( + &store, + &backend, + bucket, + &namespace, + "events", + CatalogImportRequest { + metadata_location: current_location.clone(), + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("catalog import should register table"); + let current = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + + let invalid_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + backend + .put_json( + bucket, + &invalid_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://other-warehouse/tables/table-id" + }), + ) + .await; + + assert!( + rollback_table_response( + &store, + &backend, + bucket, + &namespace, + "events", + RollbackTableRequest { + metadata_location: invalid_location, + version_token: current.version_token, + commit_id: Some("rollback-1".to_string()), + idempotency_key: None, + }, + ) + .await + .is_err() + ); + let unchanged = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + + assert_eq!(unchanged.metadata_location, current_location); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn rollback_rejects_mismatched_table_uuid_before_commit() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + ensure_table_bucket_entry(&store, bucket, true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + bucket, + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + backend + .put_json( + bucket, + ¤t_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + catalog_import_response( + &store, + &backend, + bucket, + &namespace, + "events", + CatalogImportRequest { + metadata_location: current_location.clone(), + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("catalog import should register table"); + let current = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + + let mismatched_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + backend + .put_json( + bucket, + &mismatched_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "other-table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + assert!( + rollback_table_response( + &store, + &backend, + bucket, + &namespace, + "events", + RollbackTableRequest { + metadata_location: mismatched_location, + version_token: current.version_token, + commit_id: Some("rollback-1".to_string()), + idempotency_key: None, + }, + ) + .await + .is_err() + ); + let unchanged = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current_location); + assert_eq!(unchanged.generation, current.generation); +} + +#[tokio::test] +async fn legacy_commit_rejects_mismatched_table_uuid_before_commit() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(&store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let current_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; + metadata_backend + .put_json( + "warehouse", + current_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + register_table_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: current_location.to_string(), + overwrite: false, + }, + true, + ) + .await + .expect("table should register"); + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let mismatched_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; + metadata_backend + .put_json( + "warehouse", + mismatched_location, + serde_json::json!({ + "format-version": 2, + "table-uuid": "other-table-uuid", + "location": "s3://warehouse/tables/table-id" + }), + ) + .await; + + assert!( + commit_table_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "events", + RestCommitTableRequest { + commit_id: Some("commit-1".to_string()), + idempotency_key: None, + operation: Some("append".to_string()), + expected_version_token: Some(current.version_token.clone()), + expected_metadata_location: Some(current.metadata_location.clone()), + new_metadata_location: Some(mismatched_location.to_string()), + requirements: Vec::new(), + updates: Vec::new(), + _identifier: None, + writer: Some("pyiceberg".to_string()), + }, + ) + .await + .is_err() + ); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current_location); + assert_eq!(unchanged.generation, current.generation); +} diff --git a/rustfs/src/admin/handlers/table_catalog/view.rs b/rustfs/src/admin/handlers/table_catalog/view.rs new file mode 100644 index 000000000..4c16172ef --- /dev/null +++ b/rustfs/src/admin/handlers/table_catalog/view.rs @@ -0,0 +1,120 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub struct RestListViewsHandler {} + +#[async_trait::async_trait] +impl Operation for RestListViewsHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let resource = TableCatalogResource::namespace(&warehouse, &namespace); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + let response = list_views_response(&store, &warehouse, &namespace, &req.uri).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestCreateViewHandler {} + +#[async_trait::async_trait] +impl Operation for RestCreateViewHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let resource = TableCatalogResource::namespace(&warehouse, &namespace); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::CreateTableAction).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let response = + create_view_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestLoadViewHandler {} + +#[async_trait::async_trait] +impl Operation for RestLoadViewHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let view = view_name_from_params(¶ms)?; + let resource = TableCatalogResource::view(&warehouse, &namespace, &view); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = load_view_response(&store, &metadata_backend, &warehouse, &namespace, &view).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestViewExistsHandler {} + +#[async_trait::async_trait] +impl Operation for RestViewExistsHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let view = view_name_from_params(¶ms)?; + let resource = TableCatalogResource::view(&warehouse, &namespace, &view); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + Ok(empty_response(view_exists_status(&store, &warehouse, &namespace, &view).await?)) + } +} + +pub struct RestReplaceViewHandler {} + +#[async_trait::async_trait] +impl Operation for RestReplaceViewHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let view = view_name_from_params(¶ms)?; + let resource = TableCatalogResource::view(&warehouse, &namespace, &view); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let request = read_json_body::(req.input).await?; + let metadata_backend = table_catalog_backend()?; + let store = table_catalog_store_from_backend(metadata_backend.clone())?; + let response = replace_view_response(&store, &metadata_backend, &warehouse, &namespace, &view, request).await?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestDropViewHandler {} + +#[async_trait::async_trait] +impl Operation for RestDropViewHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let namespace = namespace_from_params(¶ms)?; + let view = view_name_from_params(¶ms)?; + let resource = TableCatalogResource::view(&warehouse, &namespace, &view); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableAction).await?; + ensure_table_bucket_enabled(&warehouse).await?; + let store = table_catalog_store()?; + drop_view_in_store(&store, &warehouse, &namespace, &view).await?; + Ok(empty_response(StatusCode::NO_CONTENT)) + } +} diff --git a/rustfs/src/table_catalog.rs b/rustfs/src/table_catalog.rs deleted file mode 100644 index 0a1c79bcf..000000000 --- a/rustfs/src/table_catalog.rs +++ /dev/null @@ -1,20973 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Internal table catalog primitives for the Iceberg REST Catalog framework. -//! -//! This module intentionally does not expose HTTP handlers or mutate existing -//! S3 object behavior. It defines the stable internal boundary that later -//! catalog routes and object guards can share. - -#![allow(dead_code)] - -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt, - num::NonZeroUsize, - ops::Bound, - sync::Arc, - time::{Duration as StdDuration, Instant}, -}; - -use crate::storage_api::table::contract::http::HTTPPreconditions; -use crate::storage_api::table::contract::list::{ - ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, - ListOperations as StorageListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions, -}; -use crate::storage_api::table::contract::namespace::NamespaceLocking as StorageNamespaceLocking; -use crate::storage_api::table::contract::object::{ObjectIO as StorageObjectIO, ObjectOperations as StorageObjectOperations}; -use crate::storage_api::table::contract::range::HTTPRangeSpec; -use crate::storage_api::table::{ - BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, - BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, RUSTFS_META_BUCKET, StorageError, get_bucket_metadata, - get_lock_acquire_timeout, table_catalog_path_hash, -}; -use bytes::Bytes; -use datafusion::{ - arrow::datatypes::SchemaRef, - parquet::arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder}, -}; -use http::HeaderMap; -use metrics::{counter, histogram}; -use rustfs_filemeta::FileInfo; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use sha2::{Digest, Sha256}; -use time::{Duration, OffsetDateTime}; -use tokio::io::AsyncReadExt; -use uuid::Uuid; - -use crate::storage_api::table::{ - StorageDeletedObject as DeletedObject, StorageGetObjectReader as GetObjectReader, StorageObjectInfo as ObjectInfo, - StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader, -}; - -pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = BUCKET_TABLE_CONFIG; -pub(crate) const RESERVED_CATALOG_OBJECT_MESSAGE: &str = "Object key is reserved for the table catalog"; -pub(crate) const TABLE_BUCKET_CATALOG_TYPE: &str = "iceberg-rest"; -pub(crate) const TABLE_BUCKET_CONFIG_VERSION: u16 = 1; -pub(crate) const DEFAULT_WAREHOUSE_ID: &str = "default"; -pub(crate) const TABLE_NAMESPACE_MARKER_VERSION: u16 = 1; -pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1; -pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1; -pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1; -pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1; -pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1; -pub(crate) const TABLE_CATALOG_BACKING_MANIFEST_VERSION: u16 = 1; -pub(crate) const ENV_TABLE_CATALOG_BACKING: &str = "RUSTFS_TABLE_CATALOG_BACKING"; -pub(crate) const TABLE_CATALOG_BACKING_OBJECT: &str = "object"; -pub(crate) const TABLE_CATALOG_BACKING_DURABLE_STRONG: &str = "durable-strong"; -pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128; -pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX; -const WAREHOUSE_ROOT: &str = "warehouses"; -const NAMESPACE_ROOT: &str = "namespaces"; -const TABLE_ROOT: &str = "tables"; -const VIEW_ROOT: &str = "views"; -const NAMESPACE_MARKER_FILE: &str = "namespace.json"; -const TABLE_MARKER_FILE: &str = "table.json"; -const CURRENT_POINTER_FILE: &str = "current.json"; -const LIFECYCLE_FILE: &str = "lifecycle.json"; -const METADATA_DIR: &str = "metadata"; -const DATA_DIR: &str = "data"; -const DELETE_DIR: &str = "delete"; -const TABLE_BUCKET_ENTRY_FILE: &str = "table-bucket.json"; -const NAMESPACE_ENTRY_FILE: &str = "namespace-entry.json"; -const TABLE_ENTRY_FILE: &str = "table-entry.json"; -const VIEW_ENTRY_FILE: &str = "view-entry.json"; -const INTERNAL_CATALOG_ROOT: &str = BUCKET_TABLE_CATALOG_META_PREFIX; -const TABLE_BUCKET_ROOT: &str = BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX; -const COMMIT_LOG_ROOT: &str = "commits"; -const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency"; -const WAREHOUSE_INDEX_ROOT: &str = "warehouse-index"; -const WAREHOUSE_INDEX_STATE_FILE: &str = "state.json"; -const WAREHOUSE_INDEX_MAX_PREFIX_DEPTH: usize = 64; -const EXTERNAL_CATALOG_ROOT: &str = "external-catalog"; -const EXTERNAL_CATALOG_BRIDGE_FILE: &str = "bridge.json"; -const MAINTENANCE_ROOT: &str = "maintenance"; -const MAINTENANCE_CONFIG_FILE: &str = "config.json"; -const MAINTENANCE_JOB_ROOT: &str = "jobs"; -const MAINTENANCE_LATEST_JOB_FILE: &str = "latest.json"; -const MAINTENANCE_CURRENT_JOB_FILE: &str = "current.json"; -const MAINTENANCE_JOB_ALIAS_LATEST: &str = "latest"; -const MAINTENANCE_JOB_ALIAS_CURRENT: &str = "current"; -const TABLE_CATALOG_LIST_MAX_KEYS: usize = 1000; -const OBJECT_CATALOG_LIST_CURSOR_PREFIX: &str = "object:"; -const STRONG_CATALOG_LIST_CURSOR_PREFIX: &str = "strong:"; -const TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS: i64 = 15 * 60; -const TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS: u64 = 24 * 60 * 60; -const TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_DEFAULT_SECONDS: u64 = 15 * 60; -const TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_MAX_SECONDS: u64 = 24 * 60 * 60; -const TABLE_MAINTENANCE_SCHEDULER_AUDIT_LIMIT: usize = 10; -const TABLE_MAINTENANCE_DELETE_DISABLED_REASON: &str = "metadata delete is disabled by maintenance config"; -const TABLE_COMMIT_SLOW_LOG_THRESHOLD: StdDuration = StdDuration::from_secs(2); -const ICEBERG_MAIN_REF: &str = "main"; -const ICEBERG_MIN_SNAPSHOTS_TO_KEEP_PROPERTY: &str = "history.expire.min-snapshots-to-keep"; -const ICEBERG_MAX_SNAPSHOT_AGE_MS_PROPERTY: &str = "history.expire.max-snapshot-age-ms"; -const ICEBERG_MAX_REF_AGE_MS_PROPERTY: &str = "history.expire.max-ref-age-ms"; -const ICEBERG_REF_MIN_SNAPSHOTS_TO_KEEP_FIELD: &str = "min-snapshots-to-keep"; -const ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD: &str = "max-snapshot-age-ms"; -const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms"; -const STRONG_TABLE_CATALOG_SNAPSHOT_VERSION: u16 = 1; -const STRONG_TABLE_CATALOG_BACKING_ROOT: &str = "strong-backing"; -const STRONG_TABLE_CATALOG_SNAPSHOT_FILE: &str = "snapshot.json"; -const TABLE_CATALOG_MIGRATION_VERSION: u16 = 1; -const TABLE_CATALOG_MIGRATION_ROOT: &str = "backing-migration"; -const TABLE_CATALOG_MIGRATION_FENCE_FILE: &str = "durable-strong-fence.json"; -const TABLE_CATALOG_MIGRATION_FENCE_LOCK: &str = "durable-strong-fence.lock"; -const TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_FILE: &str = "durable-strong-global-fence.json"; -const TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_LOCK: &str = "durable-strong-global-fence.lock"; - -type CatalogListObjectsV2Info = StorageListObjectsV2Info; -type CatalogListObjectVersionsInfo = StorageListObjectVersionsInfo; -type CatalogObjectInfoOrErr = StorageObjectInfoOrErr; -type CatalogWalkOptions = StorageWalkOptions bool>; - -pub(crate) trait TableCatalogStorage: - StorageObjectIO< - Error = EcstoreError, - RangeSpec = HTTPRangeSpec, - HeaderMap = HeaderMap, - ObjectOptions = ObjectOptions, - ObjectInfo = ObjectInfo, - GetObjectReader = GetObjectReader, - PutObjectReader = PutObjReader, - > + StorageObjectOperations< - Error = EcstoreError, - ObjectInfo = ObjectInfo, - ObjectOptions = ObjectOptions, - FileInfo = FileInfo, - ObjectToDelete = ObjectToDelete, - DeletedObject = DeletedObject, - > + StorageListOperations< - Error = EcstoreError, - ListObjectsV2Info = CatalogListObjectsV2Info, - ListObjectVersionsInfo = CatalogListObjectVersionsInfo, - ObjectInfoOrErr = CatalogObjectInfoOrErr, - WalkOptions = CatalogWalkOptions, - WalkCancellation = tokio_util::sync::CancellationToken, - WalkResultSender = tokio::sync::mpsc::Sender, - > + StorageNamespaceLocking -{ -} - -impl TableCatalogStorage for T where - T: StorageObjectIO< - Error = EcstoreError, - RangeSpec = HTTPRangeSpec, - HeaderMap = HeaderMap, - ObjectOptions = ObjectOptions, - ObjectInfo = ObjectInfo, - GetObjectReader = GetObjectReader, - PutObjectReader = PutObjReader, - > + StorageObjectOperations< - Error = EcstoreError, - ObjectInfo = ObjectInfo, - ObjectOptions = ObjectOptions, - FileInfo = FileInfo, - ObjectToDelete = ObjectToDelete, - DeletedObject = DeletedObject, - > + StorageListOperations< - Error = EcstoreError, - ListObjectsV2Info = CatalogListObjectsV2Info, - ListObjectVersionsInfo = CatalogListObjectVersionsInfo, - ObjectInfoOrErr = CatalogObjectInfoOrErr, - WalkOptions = CatalogWalkOptions, - WalkCancellation = tokio_util::sync::CancellationToken, - WalkResultSender = tokio::sync::mpsc::Sender, - > + StorageNamespaceLocking -{ -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CatalogIdentifierError { - Empty, - TooLong { max: usize }, - NamespaceTooLong { max: usize }, - InvalidCharacter, - InvalidBoundary, - Ambiguous, -} - -impl fmt::Display for CatalogIdentifierError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Empty => f.write_str("catalog identifier segment is empty"), - Self::TooLong { max } => write!(f, "catalog identifier segment exceeds {max} characters"), - Self::NamespaceTooLong { max } => write!(f, "catalog namespace exceeds {max} characters"), - Self::InvalidCharacter => f.write_str("catalog identifier segment contains invalid characters"), - Self::InvalidBoundary => { - f.write_str("catalog identifier segment must start and end with a lowercase letter or digit") - } - Self::Ambiguous => f.write_str("catalog identifier segment is ambiguous"), - } - } -} - -impl std::error::Error for CatalogIdentifierError {} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum TableObjectMutationError { - ReservedCatalogObject, -} - -impl fmt::Display for TableObjectMutationError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::ReservedCatalogObject => f.write_str("object key is reserved for the table catalog"), - } - } -} - -impl std::error::Error for TableObjectMutationError {} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableBucketMarker { - pub version: u16, - pub catalog_type: &'static str, - pub reserved_prefix: &'static str, -} - -impl Default for TableBucketMarker { - fn default() -> Self { - Self { - version: TABLE_BUCKET_CONFIG_VERSION, - catalog_type: TABLE_BUCKET_CATALOG_TYPE, - reserved_prefix: TABLE_RESERVED_PREFIX, - } - } -} - -pub(crate) fn table_bucket_marker_json() -> Result, serde_json::Error> { - serde_json::to_vec(&TableBucketMarker::default()) -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogEntryState { - Active, - Deleting, - Deleted, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableBucketEntry { - pub version: u16, - pub table_bucket: String, - pub catalog_type: String, - pub warehouse_root: String, - pub state: TableCatalogEntryState, - #[serde(default)] - pub properties: BTreeMap, - pub created_at: Option, - pub updated_at: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct NamespaceEntry { - pub version: u16, - pub table_bucket: String, - pub namespace: String, - pub namespace_id: String, - pub state: TableCatalogEntryState, - #[serde(default)] - pub properties: BTreeMap, - pub created_at: Option, - pub updated_at: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableEntry { - pub version: u16, - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub table_id: String, - pub table_uuid: String, - pub format: String, - pub format_version: u16, - pub warehouse_location: String, - pub metadata_location: String, - pub version_token: String, - pub generation: u64, - pub state: TableCatalogEntryState, - #[serde(default)] - pub properties: BTreeMap, - pub created_at: Option, - pub updated_at: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct TableWarehouseIndexEntry { - version: u16, - table_bucket: String, - namespace: String, - table: String, - table_id: String, - warehouse_object_prefix: String, - state: TableCatalogEntryState, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct TableWarehouseIndexStateEntry { - version: u16, - table_bucket: String, - state: TableCatalogEntryState, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum WarehouseIndexReservation { - Created, - AlreadyReserved, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ViewEntry { - pub version: u16, - pub table_bucket: String, - pub namespace: String, - pub view: String, - pub view_id: String, - pub view_uuid: String, - pub format: String, - pub format_version: u16, - pub warehouse_location: String, - pub metadata_location: String, - pub version_token: String, - pub generation: u64, - pub state: TableCatalogEntryState, - #[serde(default)] - pub properties: BTreeMap, - pub created_at: Option, - pub updated_at: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum CommitLogStatus { - Staged, - Committed, - Failed, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct CommitLogEntry { - pub version: u16, - pub commit_id: String, - pub idempotency_key: Option, - pub table_id: String, - pub operation: String, - pub expected_version_token: String, - pub new_version_token: String, - pub previous_metadata_location: String, - pub new_metadata_location: String, - #[serde(default)] - pub requirements: Vec, - pub status: CommitLogStatus, - pub writer: Option, - pub created_at: Option, - pub updated_at: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableCommitRequest { - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub commit_id: String, - pub idempotency_key: Option, - pub operation: String, - pub expected_version_token: String, - pub expected_metadata_location: String, - pub new_metadata_location: String, - #[serde(default)] - pub requirements: Vec, - pub writer: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableCommitResult { - pub table: TableEntry, - pub commit_log: CommitLogEntry, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ExternalCatalogBridgeEntry { - pub version: u16, - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub catalog: String, - pub external_catalog_id: Option, - pub external_namespace: String, - pub external_table: String, - pub external_table_uuid: Option, - pub metadata_location: Option, - pub external_version_token: Option, - pub policy_mode: String, - pub credential_mode: String, - pub sync_mode: String, - pub rollback_strategy: String, - pub last_sync_status: Option, - pub last_synced_metadata_location: Option, - #[serde(default)] - pub properties: BTreeMap, - pub created_at: Option, - pub updated_at: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ViewCommitRequest { - pub table_bucket: String, - pub namespace: String, - pub view: String, - pub expected_version_token: String, - pub expected_metadata_location: String, - pub new_metadata_location: String, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ViewCommitResult { - pub view: ViewEntry, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableMaintenanceConfig { - pub version: u16, - #[serde(rename = "retain-recent-metadata-files")] - pub retain_recent_metadata_files: usize, - #[serde(rename = "delete-enabled")] - pub delete_enabled: bool, - #[serde(rename = "background-enabled")] - pub background_enabled: bool, - #[serde(default, rename = "worker-paused")] - pub worker_paused: bool, - #[serde( - default = "default_table_maintenance_worker_lease_timeout_seconds", - rename = "worker-lease-timeout-seconds" - )] - pub worker_lease_timeout_seconds: u64, - #[serde(default, rename = "max-retry-attempts")] - pub max_retry_attempts: u16, - #[serde(default, rename = "retry-initial-backoff-seconds")] - pub retry_initial_backoff_seconds: u64, - #[serde(default, rename = "retry-max-backoff-seconds")] - pub retry_max_backoff_seconds: u64, - #[serde(default, rename = "quarantine-enabled")] - pub quarantine_enabled: bool, - #[serde(default, rename = "quarantine-retention-seconds")] - pub quarantine_retention_seconds: u64, -} - -impl Default for TableMaintenanceConfig { - fn default() -> Self { - Self { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 0, - delete_enabled: false, - background_enabled: false, - worker_paused: false, - worker_lease_timeout_seconds: TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_DEFAULT_SECONDS, - max_retry_attempts: 0, - retry_initial_backoff_seconds: 5, - retry_max_backoff_seconds: 300, - quarantine_enabled: false, - quarantine_retention_seconds: 0, - } - } -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMaintenanceConfigSource { - #[default] - Default, - TableBucketDefault, - TableOverride, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableMaintenanceEffectiveConfig { - pub config: TableMaintenanceConfig, - pub source: TableMaintenanceConfigSource, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMaintenanceSchedulerStatus { - Ready, - Queued, - Disabled, - Paused, - Backpressured, - RetryDeferred, - Quarantined, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMaintenanceSchedulerReport { - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub table_id: String, - pub status: TableMaintenanceSchedulerStatus, - pub config_source: TableMaintenanceConfigSource, - pub background_enabled: bool, - pub worker_paused: bool, - pub delete_enabled: bool, - pub worker_lease_timeout_seconds: u64, - pub max_retry_attempts: u16, - pub retry_initial_backoff_seconds: u64, - pub retry_max_backoff_seconds: u64, - pub recommended_actions: Vec, - pub current_job: Option, - pub quarantine: TableMaintenanceSchedulerQuarantineBoundary, - pub audit_timeline: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMaintenanceSchedulerRunResult { - pub report: TableMetadataMaintenanceReport, - pub scheduler: TableMaintenanceSchedulerReport, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMaintenanceSchedulerQuarantineBoundary { - pub enabled: bool, - pub active: bool, - pub retention_seconds: u64, - pub quarantined_object_count: usize, - pub source_job_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMaintenanceAuditActor { - Scheduler, - Worker, - Operator, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMaintenanceAuditAction { - Planned, - WorkerControl, - SchedulerControl, - SchedulerQueued, - SchedulerLeaseExpired, - WorkerStarted, - WorkerHeartbeat, - WorkerLeaseExpired, - WorkerSucceeded, - WorkerFailed, - QuarantineRelease, - QuarantineRetry, - QuarantineAbandon, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMaintenanceAuditEvent { - pub timestamp: String, - pub actor: TableMaintenanceAuditActor, - pub action: TableMaintenanceAuditAction, - #[serde(default)] - pub reason: Option, - #[serde(default, rename = "before-status")] - pub before_status: Option, - #[serde(default, rename = "after-status")] - pub after_status: Option, - #[serde(default, rename = "before-quarantined-object-count")] - pub before_quarantined_object_count: Option, - #[serde(default, rename = "after-quarantined-object-count")] - pub after_quarantined_object_count: Option, - #[serde(default, rename = "recommended-actions")] - pub recommended_actions: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMaintenanceQuarantineAction { - Inspect, - Release, - Retry, - Abandon, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableMaintenanceQuarantineOperationRequest { - pub action: TableMaintenanceQuarantineAction, - #[serde(default)] - pub reason: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMaintenanceQuarantineOperationResult { - pub action: TableMaintenanceQuarantineAction, - pub report: TableMetadataMaintenanceReport, - pub scheduler: TableMaintenanceSchedulerReport, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMaintenanceSchedulerJobSummary { - pub job_id: String, - pub operation: TableMetadataMaintenanceOperation, - pub status: TableMetadataMaintenanceJobStatus, - #[serde(default, rename = "scheduler-id")] - pub scheduler_id: Option, - #[serde(default, rename = "scheduled-at")] - pub scheduled_at: Option, - pub worker_id: Option, - pub attempt: u16, - pub started_at: Option, - pub finished_at: Option, - pub heartbeat_at: Option, - pub next_retry_after: Option, - pub recommended_actions: Vec, - #[serde(default, rename = "audit-events")] - pub audit_events: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMetadataMaintenanceJob { - pub job_id: String, - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub table_id: String, - #[serde(default)] - pub operation: TableMetadataMaintenanceOperation, - #[serde(default)] - pub status: TableMetadataMaintenanceJobStatus, - #[serde(default)] - pub failure_reason: Option, - #[serde(default, rename = "recommended-actions")] - pub recommended_actions: Vec, - #[serde(default)] - pub config_source: TableMaintenanceConfigSource, - #[serde(default, rename = "scheduler-id")] - pub scheduler_id: Option, - #[serde(default, rename = "scheduler-lease-id")] - pub scheduler_lease_id: String, - #[serde(default, rename = "scheduled-at")] - pub scheduled_at: Option, - #[serde(default)] - pub worker_id: Option, - #[serde(default)] - pub lease_id: String, - #[serde(default)] - pub attempt: u16, - #[serde(default, rename = "max-retry-attempts")] - pub max_retry_attempts: u16, - #[serde(default, rename = "next-retry-after")] - pub next_retry_after: Option, - #[serde(default, rename = "quarantine-enabled")] - pub quarantine_enabled: bool, - #[serde(default, rename = "quarantine-retention-seconds")] - pub quarantine_retention_seconds: u64, - #[serde(default)] - pub heartbeat_at: Option, - #[serde(default)] - pub started_at: Option, - #[serde(default)] - pub finished_at: Option, - pub current_metadata_location: String, - pub current_generation: u64, - pub retain_recent_metadata_files: usize, - pub safety_window_seconds: i64, - pub cleanup_watermark_unix_seconds: i64, - #[serde(default)] - pub planned_metadata_file_count: usize, - #[serde(default)] - pub retained_metadata_file_count: usize, - #[serde(default)] - pub cleanup_candidate_count: usize, - #[serde(default)] - pub deletable_metadata_file_count: usize, - #[serde(default)] - pub deleted_metadata_file_count: usize, - #[serde(default)] - pub planned_object_file_count: usize, - #[serde(default)] - pub cleanup_candidate_object_count: usize, - #[serde(default)] - pub deletable_object_count: usize, - #[serde(default)] - pub deleted_object_count: usize, - #[serde(default)] - pub quarantined_object_count: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMetadataMaintenanceReport { - pub job: TableMetadataMaintenanceJob, - pub current_metadata_location: String, - pub retained_metadata_locations: Vec, - pub cleanup_candidate_locations: Vec, - pub deletable_metadata_locations: Vec, - #[serde(default, rename = "cleanup-object-candidate-locations")] - pub cleanup_object_candidate_locations: Vec, - #[serde(default, rename = "deletable-object-locations")] - pub deletable_object_locations: Vec, - #[serde(default)] - pub object_reports: Vec, - #[serde(default, rename = "object-cleanup-reports")] - pub object_cleanup_reports: Vec, - #[serde(default)] - pub referenced_object_reports: Vec, - #[serde(default, rename = "reachability-graph")] - pub reachability_graph: TableMaintenanceReachabilityGraphReport, - #[serde(default, rename = "snapshot-expiration")] - pub snapshot_expiration: Option, - #[serde(default)] - pub compaction: Option, - #[serde(default, rename = "audit-events")] - pub audit_events: Vec, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMaintenanceReachabilityGraphReport { - pub status: TableMaintenanceReachabilityGraphStatus, - pub metadata_file_count: usize, - pub manifest_list_count: usize, - pub manifest_file_count: usize, - pub data_file_count: usize, - pub delete_file_count: usize, - pub manual_review_count: usize, - pub reasons: Vec, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMaintenanceReachabilityGraphStatus { - Complete, - #[default] - ManualReviewRequired, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMaintenanceReachabilityGraphReason { - MetadataJsonParsed, - ManifestListAvroReferenced, - ManifestAvroReaderUnavailable, - DataFileCleanupDeferred, - DeleteFileCleanupDeferred, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableSnapshotExpirationConfig { - #[serde(rename = "min-snapshots-to-keep")] - pub min_snapshots_to_keep: usize, - #[serde(rename = "max-snapshot-age-ms")] - pub max_snapshot_age_ms: i64, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableSnapshotExpirationReport { - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub table_id: String, - pub current_metadata_location: String, - pub current_snapshot_id: Option, - pub config: TableSnapshotExpirationConfig, - pub expiration_watermark_ms: i64, - pub retained_snapshot_count: usize, - pub expiration_candidate_count: usize, - pub manual_review_count: usize, - #[serde(default)] - pub expired_snapshot_ids: Vec, - #[serde(default)] - pub committed_metadata_location: Option, - pub snapshot_reports: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableSnapshotExpirationSnapshotReport { - pub snapshot_id: Option, - pub sequence_number: Option, - pub timestamp_ms: Option, - pub manifest_list: Option, - pub state: TableSnapshotExpirationSnapshotState, - pub reasons: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableSnapshotExpirationSnapshotState { - Retained, - ExpirationCandidate, - ManualReviewRequired, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableSnapshotExpirationReason { - CurrentSnapshot, - MinSnapshotsToKeep, - ProtectedSnapshotRef, - UserDefinedSnapshotRef, - SnapshotRefRetentionConflict, - TableRetentionPropertyConflict, - MissingSnapshotId, - MissingSnapshotTimestamp, - SnapshotAgeWithinRetention, - SnapshotAgeExpired, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TableCompactionPlanningConfig { - #[serde(rename = "target-file-size-bytes")] - pub target_file_size_bytes: u64, - #[serde(rename = "small-file-threshold-bytes")] - pub small_file_threshold_bytes: u64, - #[serde(rename = "min-input-files")] - pub min_input_files: usize, - #[serde(rename = "max-rewrite-bytes-per-job")] - pub max_rewrite_bytes_per_job: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableCompactionPlanningReport { - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub table_id: String, - pub current_metadata_location: String, - pub current_snapshot_id: Option, - pub config: TableCompactionPlanningConfig, - pub status: TableCompactionPlanningStatus, - pub candidate_file_count: usize, - pub rewrite_group_count: usize, - pub manual_review_count: usize, - #[serde(default, rename = "committed-metadata-location")] - pub committed_metadata_location: Option, - #[serde(default, rename = "row-level-planning")] - pub row_level_planning: TableRowLevelMaintenancePlanningReport, - #[serde(default, rename = "rewrite-groups")] - pub rewrite_groups: Vec, - pub snapshot_reports: Vec, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableRowLevelMaintenancePlanningReport { - pub status: TableRowLevelMaintenancePlanningStatus, - #[serde(rename = "delete-file-count")] - pub delete_file_count: usize, - #[serde(rename = "position-delete-file-count")] - pub position_delete_file_count: usize, - #[serde(rename = "equality-delete-file-count")] - pub equality_delete_file_count: usize, - #[serde(rename = "manual-review-count")] - pub manual_review_count: usize, - pub reasons: Vec, - #[serde(rename = "delete-files")] - pub delete_files: Vec, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableRowLevelMaintenancePlanningStatus { - #[default] - NoDeleteFiles, - ManualReviewRequired, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableRowLevelMaintenancePlanningReason { - PositionDeleteFile, - EqualityDeleteFile, - DeleteFileRewriteUnsupported, - MissingDeleteFile, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableRowLevelDeleteFilePlanningReport { - #[serde(rename = "file-location")] - pub file_location: String, - pub content: TableRowLevelDeleteFileContent, - #[serde(rename = "object-exists")] - pub object_exists: bool, - #[serde(default, rename = "record-count", skip_serializing_if = "Option::is_none")] - pub record_count: Option, - #[serde(default, rename = "file-size-bytes", skip_serializing_if = "Option::is_none")] - pub file_size_bytes: Option, - #[serde(default, rename = "sequence-number", skip_serializing_if = "Option::is_none")] - pub sequence_number: Option, - #[serde(default, rename = "file-sequence-number", skip_serializing_if = "Option::is_none")] - pub file_sequence_number: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableRowLevelDeleteFileContent { - PositionDelete, - EqualityDelete, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableCompactionRewriteGroup { - pub group_id: String, - #[serde(default, rename = "sort-order-id", skip_serializing_if = "Option::is_none")] - pub sort_order_id: Option, - #[serde(rename = "input-file-locations")] - pub input_file_locations: Vec, - #[serde(rename = "input-file-count")] - pub input_file_count: usize, - #[serde(rename = "input-bytes")] - pub input_bytes: u64, - #[serde(default, rename = "output-file-location")] - pub output_file_location: Option, - #[serde(default, rename = "output-bytes")] - pub output_bytes: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableCompactionSnapshotReport { - pub snapshot_id: Option, - pub manifest_list: Option, - pub status: TableCompactionPlanningStatus, - pub reasons: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCompactionPlanningStatus { - NoCandidates, - RewriteCandidates, - Committed, - ManualReviewRequired, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCompactionPlanningReason { - ManifestList, - ManifestFile, - SmallDataFile, - RewriteGroup, - CompactionCommitted, - ManifestAvroReaderUnavailable, - MissingCurrentSnapshot, - MissingManifestList, - MissingDataFile, - DeleteFile, - PositionDeleteFile, - EqualityDeleteFile, - RowLevelRewriteUnsupported, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMetadataMaintenanceOperation { - #[default] - DryRun, - Delete, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMetadataMaintenanceJobStatus { - NotYetRun, - Queued, - Running, - #[default] - Successful, - Failed, - Disabled, - Paused, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMaintenanceRecommendedAction { - NoActionRequired, - RunMaintenanceWorker, - ReviewAndRunDelete, - ReviewQuarantine, - EnableDelete, - EnableBackgroundMaintenance, - ResumeMaintenanceWorker, - WaitForRetryBackoff, - WaitForActiveWorker, - InvestigateFailure, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMetadataMaintenanceObjectState { - Retained, - PendingSafetyWindow, - Deletable, - Deleted, - ManualReviewRequired, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMetadataMaintenanceReason { - CurrentMetadata, - MetadataLog, - ProtectedSnapshotRef, - RecentMetadata, - NoCurrentReachability, - SafetyWindowPending, - SafetyWindowSatisfied, - DeletedByMaintenance, - ManifestList, - ManifestFile, - DataFile, - DeleteFile, - UnsupportedManifestAvro, - UnreadableMetadata, - QuarantineEnabled, - RetryScheduled, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMetadataMaintenanceObjectKind { - MetadataFile, - ManifestList, - ManifestFile, - DataFile, - DeleteFile, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMetadataMaintenanceObjectReport { - pub metadata_location: String, - pub state: TableMetadataMaintenanceObjectState, - pub reasons: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMetadataMaintenanceObjectCleanupReport { - pub object_location: String, - pub object_kind: TableMetadataMaintenanceObjectKind, - pub state: TableMetadataMaintenanceObjectState, - pub reasons: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMetadataMaintenanceReferencedObjectReport { - pub object_location: String, - pub object_kind: TableMetadataMaintenanceObjectKind, - pub state: TableMetadataMaintenanceObjectState, - pub reasons: Vec, -} - -struct TableMaintenanceHeartbeatRef<'a> { - table_bucket: &'a str, - namespace: &'a str, - table: &'a str, - job_id: &'a str, - lease_id: &'a str, - worker_id: &'a str, -} - -struct TableMaintenancePreflightContext<'a> { - table_bucket: &'a str, - namespace: &'a Namespace, - table: &'a IdentifierSegment, - entry: &'a TableEntry, -} - -struct TableMaintenanceWorkerControlReport<'a> { - table_bucket: &'a str, - namespace: &'a Namespace, - table: &'a IdentifierSegment, - entry: &'a TableEntry, - worker_id: String, - effective: &'a TableMaintenanceEffectiveConfig, - status: TableMetadataMaintenanceJobStatus, - reason: &'a str, - now: OffsetDateTime, -} - -struct TableMaintenanceSchedulerControlReport<'a> { - table_bucket: &'a str, - namespace: &'a Namespace, - table: &'a IdentifierSegment, - entry: &'a TableEntry, - scheduler_id: String, - effective: &'a TableMaintenanceEffectiveConfig, - status: TableMetadataMaintenanceJobStatus, - reason: &'a str, - now: OffsetDateTime, -} - -enum TableMaintenanceWorkerPreflight { - Ready { - effective: TableMaintenanceEffectiveConfig, - queued: Option>, - }, - Complete(Box), -} - -enum TableMaintenanceSchedulerPreflight { - Ready(TableMaintenanceEffectiveConfig), - Complete(Box), -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogExport { - pub table_bucket: TableBucketEntry, - pub namespace: NamespaceEntry, - pub table: TableEntry, - pub backing_manifest: TableCatalogBackingManifest, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogBackingManifest { - pub version: u16, - pub current: TableCatalogBackingProfile, - pub migration: TableCatalogBackingMigrationPlan, - pub ha: TableCatalogHaPolicy, - pub scale_validation: TableCatalogScaleValidation, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogBackingProfile { - pub kind: TableCatalogBackingKind, - pub authority: TableCatalogAuthority, - pub consistency: TableCatalogConsistencyMode, - pub durability: TableCatalogDurabilityMode, - pub current_pointer_path: String, - pub wal: TableCatalogWalState, - pub snapshot: TableCatalogSnapshotState, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogBackingKind { - ObjectBacked, - StrongKvWal, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogAuthority { - RustfsSysObject, - LinearizableMetadataKv, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogConsistencyMode { - ConditionalObjectCas, - LinearizableCas, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogDurabilityMode { - StagedCommitLogBeforePointerUpdate, - WalBeforeStateMachineApply, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogWalState { - pub status: TableCatalogWalStatus, - pub commit_log_prefix: String, - pub idempotency_index_prefix: String, - pub committed_generation: u64, - pub staged_before_table_update_count: usize, - pub finalization_required_count: usize, - pub idempotency_repair_required_count: usize, - pub manual_review_count: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogWalStatus { - Recoverable, - RecoveryRequired, - ManualReviewRequired, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogSnapshotState { - pub export_api: String, - pub includes_table_bucket: bool, - pub includes_namespace: bool, - pub includes_table_pointer: bool, - pub includes_backing_manifest: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogBackingMigrationPlan { - pub source_kind: TableCatalogBackingKind, - pub target_kind: TableCatalogBackingKind, - pub status: TableCatalogBackingMigrationStatus, - pub required_steps: Vec, - pub blockers: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogBackingMigrationDryRunReport { - pub table_bucket: String, - pub source_kind: TableCatalogBackingKind, - pub target_kind: TableCatalogBackingKind, - pub status: TableCatalogBackingMigrationStatus, - pub namespace_count: usize, - pub table_count: usize, - pub view_count: usize, - pub commit_log_count: usize, - pub idempotency_index_count: usize, - pub warehouse_prefix_count: usize, - pub warehouse_index_ready: bool, - pub object_backed_writes_fenced: bool, - pub ready_to_enable_durable_strong: bool, - pub blockers: Vec, - pub recommended_actions: Vec, - pub rollback: TableCatalogBackingRollbackPlan, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogBackingMigrationExecutionReport { - pub table_bucket: String, - pub source_kind: TableCatalogBackingKind, - pub target_kind: TableCatalogBackingKind, - pub status: TableCatalogBackingMigrationExecutionStatus, - pub namespace_count: usize, - pub table_count: usize, - pub view_count: usize, - pub commit_log_count: usize, - pub idempotency_index_count: usize, - pub source_fingerprint: String, - pub target_snapshot_etag: String, - pub object_backed_writes_fenced: bool, - pub ready_to_enable_durable_strong: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogBackingMigrationExecutionStatus { - SnapshotMaterialized, - SnapshotAlreadyMaterialized, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogBackingMigrationCancelReport { - pub table_bucket: String, - pub status: TableCatalogBackingMigrationCancelStatus, - pub object_backed_writes_fenced: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogBackingMigrationCancelStatus { - FenceReleased, - NoMigrationFence, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogBackingMigrationStatus { - ReadyToSnapshot, - SnapshotMaterialized, - RecoveryRequired, - ManualReviewRequired, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogBackingMigrationStep { - SnapshotCatalogExport, - ReplayCommitLog, - VerifyCurrentPointer, - EnableSingleWriterFencing, - CutOverLinearizableReads, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogBackingMigrationBlocker { - CommitRecoveryRequired, - CommitManualReviewRequired, - WarehouseIndexBackfillRequired, - DuplicateWarehousePrefix, - DurableStrongSnapshotChanged, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogBackingMigrationAction { - RunCatalogRecovery, - BackfillWarehouseIndex, - ReviewDuplicateWarehousePrefixes, - SnapshotObjectBackedCatalog, - EnableDurableStrongBacking, - VerifyDurableStrongSnapshot, - SnapshotRemainingTableBuckets, - ReviewDurableStrongSnapshot, - KeepObjectBackedRollbackConfig, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogBackingRollbackPlan { - pub backing_config_key: &'static str, - pub current_backing_value: &'static str, - pub rollback_backing_value: &'static str, - pub preserves_object_backed_catalog: bool, - pub requires_operator_restart: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogHaPolicy { - pub writer_region_model: TableCatalogHaWriterModel, - pub read_replica_strategy: TableCatalogReadReplicaStrategy, - pub commit_read_requirement: TableCatalogCommitReadRequirement, - pub active_active_supported: bool, - pub failover_requires_operator_promotion: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogHaWriterModel { - SingleActiveWriterRegion, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogReadReplicaStrategy { - ReadOnlyReplicasForListAndLoad, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogCommitReadRequirement { - LinearizableLeaderRead, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogScaleValidation { - pub status: TableCatalogScaleValidationStatus, - pub benchmark_required: bool, - pub required_scenarios: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogScaleValidationStatus { - MatrixPublished, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogScaleValidationScenario { - ConcurrentCommitCas, - CommitLogRecoveryReplay, - MigrationSnapshotReplay, - ReadReplicaStaleReadGuard, - ClientConformanceMatrix, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct TableDataPlaneResource { - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub table_id: String, - pub warehouse_object_prefix: String, -} - -impl TableDataPlaneResource { - pub(crate) fn catalog_resource_object(&self) -> String { - let namespace = Namespace::parse(&self.namespace) - .map(|namespace| namespace.storage_id()) - .unwrap_or_else(|_| self.namespace.clone()); - format!("{NAMESPACE_ROOT}/{namespace}/{TABLE_ROOT}/{}", self.table) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableMetadataPointerStatus { - Valid, - MissingObject, - InvalidLocation, - InvalidJson, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogRecoveryStatus { - Healthy, - Recoverable, - ManualReviewRequired, - ReadOnlyRecommended, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCatalogRecoveryAction { - RunCommitRecovery, - RetryCommit, - RestoreCurrentMetadataObject, - FixCurrentMetadataJson, - MoveCurrentMetadataInsideTable, - ReviewCommitLog, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCatalogDiagnosticsReport { - pub catalog: TableCatalogExport, - pub current_metadata_status: TableMetadataPointerStatus, - pub recovery_status: TableCatalogRecoveryStatus, - pub recommended_actions: Vec, - pub commit_recovery: TableCommitRecoveryReport, - pub backing_manifest: TableCatalogBackingManifest, - pub orphan_metadata_candidate_locations: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCommitRecoveryState { - Committed, - StagedBeforeTableUpdate, - FinalizationRequired, - IdempotencyIndexRepairRequired, - ManualReview, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub(crate) enum TableCommitIdempotencyIndexStatus { - NotRequired, - Missing, - Matches, - Stale, - Conflicting, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCommitRecoveryEntry { - pub commit_id: String, - pub idempotency_key: Option, - pub operation: String, - pub status: CommitLogStatus, - pub recovery_state: TableCommitRecoveryState, - pub previous_metadata_location: String, - pub new_metadata_location: String, - pub expected_version_token: String, - pub new_version_token: String, - pub idempotency_index_present: bool, - pub idempotency_index_status: TableCommitIdempotencyIndexStatus, - pub reason: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableCommitRecoveryReport { - pub table_bucket: String, - pub namespace: String, - pub table: String, - pub table_id: String, - pub current_metadata_location: String, - pub current_version_token: String, - pub current_generation: u64, - pub commits: Vec, - pub staged_before_table_update_count: usize, - pub finalization_required_count: usize, - pub idempotency_repair_required_count: usize, - pub manual_review_count: usize, - pub finalized_count: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum TableCatalogStoreError { - NotFound(String), - Conflict(String), - Invalid(String), - Internal(String), -} - -impl fmt::Display for TableCatalogStoreError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::NotFound(message) => write!(f, "table catalog entry not found: {message}"), - Self::Conflict(message) => write!(f, "table catalog conflict: {message}"), - Self::Invalid(message) => write!(f, "invalid table catalog entry: {message}"), - Self::Internal(message) => write!(f, "table catalog store error: {message}"), - } - } -} - -impl std::error::Error for TableCatalogStoreError {} - -pub(crate) type TableCatalogStoreResult = Result; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum TableCatalogBackingMode { - ObjectBacked, - DurableStrong, -} - -impl TableCatalogBackingMode { - pub(crate) fn from_env() -> TableCatalogStoreResult { - match std::env::var(ENV_TABLE_CATALOG_BACKING) { - Ok(value) => Self::parse(&value), - Err(std::env::VarError::NotPresent) => Ok(Self::ObjectBacked), - Err(std::env::VarError::NotUnicode(_)) => Err(TableCatalogStoreError::Invalid(format!( - "{ENV_TABLE_CATALOG_BACKING} must be valid UTF-8" - ))), - } - } - - fn parse(value: &str) -> TableCatalogStoreResult { - match value.trim() { - "" | TABLE_CATALOG_BACKING_OBJECT => Ok(Self::ObjectBacked), - TABLE_CATALOG_BACKING_DURABLE_STRONG => Ok(Self::DurableStrong), - value => Err(TableCatalogStoreError::Invalid(format!( - "unsupported table catalog backing {value}; expected {TABLE_CATALOG_BACKING_OBJECT} or {TABLE_CATALOG_BACKING_DURABLE_STRONG}" - ))), - } - } - - pub(crate) fn as_str(self) -> &'static str { - match self { - Self::ObjectBacked => TABLE_CATALOG_BACKING_OBJECT, - Self::DurableStrong => TABLE_CATALOG_BACKING_DURABLE_STRONG, - } - } -} - -fn normalize_warehouse_object_prefix(object_prefix: &str, max_prefix_depth: Option) -> TableCatalogStoreResult { - let object_prefix = object_prefix.strip_suffix('/').unwrap_or(object_prefix); - if object_prefix.is_empty() { - return Err(TableCatalogStoreError::Invalid( - "table warehouse location must include an object prefix".to_string(), - )); - } - if object_prefix.contains('\\') { - return Err(TableCatalogStoreError::Invalid( - "table warehouse location contains an invalid path separator".to_string(), - )); - } - let mut segment_count = 0; - for segment in object_prefix.split('/') { - segment_count += 1; - if segment.is_empty() || segment == "." || segment == ".." { - return Err(TableCatalogStoreError::Invalid( - "table warehouse location contains an invalid path segment".to_string(), - )); - } - } - if max_prefix_depth.is_some_and(|max_prefix_depth| segment_count > max_prefix_depth) { - return Err(TableCatalogStoreError::Invalid( - "table warehouse location exceeds the maximum prefix depth".to_string(), - )); - } - - let mut normalized = object_prefix.to_string(); - normalized.push('/'); - Ok(normalized) -} - -fn warehouse_object_prefix_from_location( - table_bucket: &str, - warehouse_location: &str, - max_prefix_depth: Option, -) -> TableCatalogStoreResult { - let location = warehouse_location - .strip_prefix("s3://") - .ok_or_else(|| TableCatalogStoreError::Invalid("table warehouse location must be an s3 URI".to_string()))?; - let (bucket, object_prefix) = location - .split_once('/') - .ok_or_else(|| TableCatalogStoreError::Invalid("table warehouse location must include an object prefix".to_string()))?; - if bucket != table_bucket { - return Err(TableCatalogStoreError::Invalid( - "table warehouse location must be inside the table bucket".to_string(), - )); - } - normalize_warehouse_object_prefix(object_prefix, max_prefix_depth) -} - -fn table_warehouse_object_prefix_from_location(table_bucket: &str, warehouse_location: &str) -> TableCatalogStoreResult { - warehouse_object_prefix_from_location(table_bucket, warehouse_location, Some(WAREHOUSE_INDEX_MAX_PREFIX_DEPTH)) -} - -fn view_warehouse_object_prefix_from_location(table_bucket: &str, warehouse_location: &str) -> TableCatalogStoreResult { - warehouse_object_prefix_from_location(table_bucket, warehouse_location, None) -} - -pub(crate) fn validate_table_warehouse_location(table_bucket: &str, warehouse_location: &str) -> TableCatalogStoreResult<()> { - table_warehouse_object_prefix_from_location(table_bucket, warehouse_location).map(|_| ()) -} - -pub(crate) fn validate_view_warehouse_location(table_bucket: &str, warehouse_location: &str) -> TableCatalogStoreResult<()> { - view_warehouse_object_prefix_from_location(table_bucket, warehouse_location).map(|_| ()) -} - -pub(crate) fn table_warehouse_object_prefix(entry: &TableEntry) -> TableCatalogStoreResult { - table_warehouse_object_prefix_from_location(&entry.table_bucket, &entry.warehouse_location) -} - -fn table_warehouse_index_entry(entry: &TableEntry) -> TableCatalogStoreResult { - Ok(TableWarehouseIndexEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: entry.table_bucket.clone(), - namespace: entry.namespace.clone(), - table: entry.table.clone(), - table_id: entry.table_id.clone(), - warehouse_object_prefix: table_warehouse_object_prefix(entry)?, - state: entry.state.clone(), - }) -} - -fn table_warehouse_data_dir_path(entry: &TableEntry) -> TableCatalogStoreResult { - Ok(format!("{}{}", table_warehouse_object_prefix(entry)?, DATA_DIR)) -} - -fn table_object_s3_location(table_bucket: &str, object_key: &str) -> String { - format!("s3://{table_bucket}/{object_key}") -} - -fn metadata_warehouse_location( - table_bucket: &str, - metadata_location: &str, - metadata_object: &TableCatalogObject, - validate_location: fn(&str, &str) -> TableCatalogStoreResult<()>, -) -> TableCatalogStoreResult> { - let metadata: serde_json::Value = serde_json::from_slice(&metadata_object.data) - .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to parse new metadata {metadata_location}: {err}")))?; - let Some(location) = metadata.get("location").and_then(serde_json::Value::as_str) else { - return Ok(None); - }; - validate_location(table_bucket, location)?; - Ok(Some(location.to_string())) -} - -fn table_metadata_warehouse_location( - table_bucket: &str, - metadata_location: &str, - metadata_object: &TableCatalogObject, -) -> TableCatalogStoreResult> { - metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_table_warehouse_location) -} - -fn view_metadata_warehouse_location( - table_bucket: &str, - metadata_location: &str, - metadata_object: &TableCatalogObject, -) -> TableCatalogStoreResult> { - metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_view_warehouse_location) -} - -fn warehouse_index_candidate_prefixes(object: &str) -> Vec<&str> { - let mut prefixes = Vec::new(); - for (index, byte) in object.as_bytes().iter().enumerate() { - if *byte == b'/' { - prefixes.push(&object[..=index]); - if prefixes.len() >= WAREHOUSE_INDEX_MAX_PREFIX_DEPTH { - break; - } - } - } - prefixes.reverse(); - prefixes -} - -fn table_data_plane_resource_from_entry(table: TableEntry, warehouse_object_prefix: String) -> TableDataPlaneResource { - TableDataPlaneResource { - table_bucket: table.table_bucket, - namespace: table.namespace, - table: table.table, - table_id: table.table_id, - warehouse_object_prefix, - } -} - -pub(crate) async fn table_data_plane_resource_for_object( - store: &S, - bucket: &str, - object: &str, -) -> TableCatalogStoreResult> -where - S: TableCatalogStore + ?Sized, -{ - store.resolve_table_data_plane_resource(bucket, object).await -} - -async fn scan_table_data_plane_resource_for_object( - store: &S, - bucket: &str, - object: &str, -) -> TableCatalogStoreResult> -where - S: TableCatalogStore + ?Sized, -{ - if bucket.is_empty() || object.is_empty() { - return Ok(None); - } - - let Some(table_bucket) = store.get_table_bucket(bucket).await? else { - return Ok(None); - }; - if table_bucket.state != TableCatalogEntryState::Active { - return Ok(None); - } - - let mut matched: Option = None; - for namespace in store.list_namespaces(bucket).await? { - if namespace.state != TableCatalogEntryState::Active { - continue; - } - for table in store.list_tables(bucket, &namespace.namespace).await? { - if table.state != TableCatalogEntryState::Active { - continue; - } - let Ok(warehouse_object_prefix) = table_warehouse_object_prefix(&table) else { - continue; - }; - if !object.starts_with(&warehouse_object_prefix) { - continue; - } - if matched - .as_ref() - .is_some_and(|current| current.warehouse_object_prefix.len() >= warehouse_object_prefix.len()) - { - continue; - } - matched = Some(table_data_plane_resource_from_entry(table, warehouse_object_prefix)); - } - } - - Ok(matched) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct TableCatalogListPage { - pub entries: Vec, - pub next_cursor: Option, -} - -fn finish_catalog_list_page( - mut entries: Vec, - limit: NonZeroUsize, - cursor_prefix: &str, - key: F, -) -> TableCatalogListPage -where - F: Fn(&T) -> &str, -{ - let next_cursor = if entries.len() > limit.get() { - entries.truncate(limit.get()); - entries.last().map(|entry| format!("{cursor_prefix}{}", key(entry))) - } else { - None - }; - TableCatalogListPage { entries, next_cursor } -} - -fn catalog_list_page_from_entries( - mut entries: Vec, - cursor: Option<&str>, - limit: NonZeroUsize, - key: F, -) -> TableCatalogListPage -where - F: Fn(&T) -> &str, -{ - entries.sort_by(|left, right| key(left).cmp(key(right))); - let start = cursor.map_or(0, |cursor| entries.partition_point(|entry| key(entry) <= cursor)); - let entries = entries.into_iter().skip(start).take(limit.get().saturating_add(1)).collect(); - finish_catalog_list_page(entries, limit, "", key) -} - -fn catalog_list_cursor<'a>(cursor: Option<&'a str>, prefix: &str) -> TableCatalogStoreResult> { - cursor - .map(|cursor| { - cursor - .strip_prefix(prefix) - .filter(|cursor| !cursor.is_empty()) - .ok_or_else(|| { - TableCatalogStoreError::Invalid("page cursor does not match the active table catalog backing".to_string()) - }) - }) - .transpose() -} - -#[async_trait::async_trait] -pub(crate) trait TableCatalogStore: Send + Sync { - async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult>; - - async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()>; - - async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()>; - - async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult>; - - async fn list_namespaces_page( - &self, - table_bucket: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - Ok(catalog_list_page_from_entries( - self.list_namespaces(table_bucket).await?, - cursor, - limit, - |entry| &entry.namespace, - )) - } - - async fn get_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; - - async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()>; - - async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()>; - - async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()>; - - async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; - - async fn list_tables_page( - &self, - table_bucket: &str, - namespace: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - Ok(catalog_list_page_from_entries( - self.list_tables(table_bucket, namespace).await?, - cursor, - limit, - |entry| &entry.table, - )) - } - - async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult>; - - async fn resolve_table_data_plane_resource( - &self, - table_bucket: &str, - object: &str, - ) -> TableCatalogStoreResult> { - scan_table_data_plane_resource_for_object(self, table_bucket, object).await - } - - async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult; - - async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()>; - - async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()>; - - async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; - - async fn list_views_page( - &self, - table_bucket: &str, - namespace: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - Ok(catalog_list_page_from_entries( - self.list_views(table_bucket, namespace).await?, - cursor, - limit, - |entry| &entry.view, - )) - } - - async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult>; - - async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult; - - async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()>; - - async fn get_commit_by_id( - &self, - table_bucket: &str, - table_id: &str, - commit_id: &str, - ) -> TableCatalogStoreResult>; - - async fn get_commit_by_idempotency_key( - &self, - table_bucket: &str, - table_id: &str, - idempotency_key: &str, - ) -> TableCatalogStoreResult>; -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct TableCatalogObject { - pub data: Vec, - pub etag: Option, - pub mod_time: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct TableCatalogObjectMetadata { - pub etag: Option, - pub mod_time: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct TableCatalogObjectListPage { - pub objects: Vec, - pub is_truncated: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum TableCatalogPutPrecondition { - Any, - IfAbsent, - IfMatch(String), -} - -#[async_trait::async_trait] -pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { - async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; - - async fn read_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - self.read_object(bucket, object).await - } - - async fn object_metadata(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - Ok(self - .read_object(bucket, object) - .await? - .map(|object| TableCatalogObjectMetadata { - etag: object.etag, - mod_time: object.mod_time, - })) - } - - async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult; - - async fn put_object( - &self, - bucket: &str, - object: &str, - data: Vec, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()>; - - async fn put_object_unlocked( - &self, - bucket: &str, - object: &str, - data: Vec, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> { - self.put_object(bucket, object, data, precondition).await - } - - async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()>; - - async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { - self.delete_object(bucket, object).await - } - - async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult>; - - async fn list_objects_page( - &self, - bucket: &str, - prefix: &str, - start_after: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult { - let mut objects = self.list_objects(bucket, prefix).await?; - objects.sort(); - let start = start_after.map_or(0, |cursor| objects.partition_point(|object| object.as_str() <= cursor)); - let mut objects = objects - .into_iter() - .skip(start) - .take(limit.get().saturating_add(1)) - .collect::>(); - let is_truncated = objects.len() > limit.get(); - objects.truncate(limit.get()); - Ok(TableCatalogObjectListPage { objects, is_truncated }) - } - - async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - self.acquire_write_lock(bucket, object).await - } - - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct TableCatalogObjectPaths { - catalog_root: &'static str, -} - -impl Default for TableCatalogObjectPaths { - fn default() -> Self { - Self { - catalog_root: INTERNAL_CATALOG_ROOT, - } - } -} - -impl TableCatalogObjectPaths { - pub fn table_bucket_entries_prefix(&self) -> String { - format!("{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT) - } - - pub fn table_bucket_entry_path(&self, table_bucket: &str) -> String { - format!("{}{}", self.table_bucket_root_prefix(table_bucket), TABLE_BUCKET_ENTRY_FILE) - } - - pub fn table_bucket_maintenance_config_path(&self, table_bucket: &str) -> String { - format!( - "{}{MAINTENANCE_ROOT}/{MAINTENANCE_CONFIG_FILE}", - self.table_bucket_root_prefix(table_bucket) - ) - } - - pub fn namespace_entries_prefix(&self, table_bucket: &str) -> String { - format!("{}{}/", self.table_bucket_root_prefix(table_bucket), NAMESPACE_ROOT) - } - - pub fn namespace_entry_path(&self, table_bucket: &str, namespace: &Namespace) -> String { - format!( - "{}{}/{}", - self.namespace_entries_prefix(table_bucket), - namespace.storage_id(), - NAMESPACE_ENTRY_FILE - ) - } - - pub fn table_entries_prefix(&self, table_bucket: &str, namespace: &Namespace) -> String { - format!( - "{}{}/{}/", - self.namespace_entries_prefix(table_bucket), - namespace.storage_id(), - TABLE_ROOT - ) - } - - pub fn table_entry_path(&self, table_bucket: &str, namespace: &Namespace, table: &IdentifierSegment) -> String { - format!( - "{}{}/{}", - self.table_entries_prefix(table_bucket, namespace), - table.as_str(), - TABLE_ENTRY_FILE - ) - } - - pub fn external_catalog_bridge_path(&self, table_bucket: &str, namespace: &Namespace, table: &IdentifierSegment) -> String { - format!( - "{}{}/{EXTERNAL_CATALOG_ROOT}/{EXTERNAL_CATALOG_BRIDGE_FILE}", - self.table_entries_prefix(table_bucket, namespace), - table.as_str() - ) - } - - pub fn view_entries_prefix(&self, table_bucket: &str, namespace: &Namespace) -> String { - format!("{}{}/{}/", self.namespace_entries_prefix(table_bucket), namespace.storage_id(), VIEW_ROOT) - } - - pub fn view_entry_path(&self, table_bucket: &str, namespace: &Namespace, view: &IdentifierSegment) -> String { - format!( - "{}{}/{}", - self.view_entries_prefix(table_bucket, namespace), - view.as_str(), - VIEW_ENTRY_FILE - ) - } - - pub fn table_maintenance_config_path( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - table_id: &str, - ) -> String { - format!( - "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CONFIG_FILE}", - self.table_entries_prefix(table_bucket, namespace), - table.as_str(), - table_catalog_path_hash(table_id) - ) - } - - pub fn table_maintenance_job_path( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - table_id: &str, - job_id: &str, - ) -> String { - format!( - "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_JOB_ROOT}/{}.json", - self.table_entries_prefix(table_bucket, namespace), - table.as_str(), - table_catalog_path_hash(table_id), - table_catalog_path_hash(job_id) - ) - } - - pub fn table_maintenance_jobs_prefix( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - table_id: &str, - ) -> String { - format!( - "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_JOB_ROOT}/", - self.table_entries_prefix(table_bucket, namespace), - table.as_str(), - table_catalog_path_hash(table_id) - ) - } - - pub fn table_maintenance_latest_job_path( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - table_id: &str, - ) -> String { - format!( - "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_LATEST_JOB_FILE}", - self.table_entries_prefix(table_bucket, namespace), - table.as_str(), - table_catalog_path_hash(table_id) - ) - } - - pub fn table_maintenance_current_job_path( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - table_id: &str, - ) -> String { - format!( - "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CURRENT_JOB_FILE}", - self.table_entries_prefix(table_bucket, namespace), - table.as_str(), - table_catalog_path_hash(table_id) - ) - } - - pub fn commit_log_entry_path(&self, table_bucket: &str, table_id: &str, commit_id: &str) -> String { - format!( - "{}{}/{}/{}.json", - self.table_bucket_root_prefix(table_bucket), - COMMIT_LOG_ROOT, - table_catalog_path_hash(table_id), - table_catalog_path_hash(commit_id) - ) - } - - pub fn commit_log_entries_prefix(&self, table_bucket: &str, table_id: &str) -> String { - format!( - "{}{}/{}/", - self.table_bucket_root_prefix(table_bucket), - COMMIT_LOG_ROOT, - table_catalog_path_hash(table_id) - ) - } - - pub fn commit_idempotency_entry_path(&self, table_bucket: &str, table_id: &str, idempotency_key: &str) -> String { - format!( - "{}{}/{}/{}.json", - self.table_bucket_root_prefix(table_bucket), - COMMIT_IDEMPOTENCY_ROOT, - table_catalog_path_hash(table_id), - table_catalog_path_hash(idempotency_key) - ) - } - - pub fn commit_idempotency_entries_prefix(&self, table_bucket: &str, table_id: &str) -> String { - format!( - "{}{}/{}/", - self.table_bucket_root_prefix(table_bucket), - COMMIT_IDEMPOTENCY_ROOT, - table_catalog_path_hash(table_id) - ) - } - - pub fn warehouse_index_state_path(&self, table_bucket: &str) -> String { - format!( - "{}{}/{}", - self.table_bucket_root_prefix(table_bucket), - WAREHOUSE_INDEX_ROOT, - WAREHOUSE_INDEX_STATE_FILE - ) - } - - pub fn warehouse_index_entry_path(&self, table_bucket: &str, warehouse_object_prefix: &str) -> String { - format!( - "{}{}/{}.json", - self.table_bucket_root_prefix(table_bucket), - WAREHOUSE_INDEX_ROOT, - table_catalog_path_hash(warehouse_object_prefix) - ) - } - - pub fn backing_migration_fence_path(&self, table_bucket: &str) -> String { - format!( - "{}{}/{}", - self.table_bucket_root_prefix(table_bucket), - TABLE_CATALOG_MIGRATION_ROOT, - TABLE_CATALOG_MIGRATION_FENCE_FILE - ) - } - - pub fn backing_migration_fence_lock_path(&self, table_bucket: &str) -> String { - format!( - "{}{}/{}", - self.table_bucket_root_prefix(table_bucket), - TABLE_CATALOG_MIGRATION_ROOT, - TABLE_CATALOG_MIGRATION_FENCE_LOCK - ) - } - - pub fn backing_migration_global_fence_path(&self) -> String { - format!( - "{}/{}/{}", - self.catalog_root, TABLE_CATALOG_MIGRATION_ROOT, TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_FILE - ) - } - - pub fn backing_migration_global_fence_lock_path(&self) -> String { - format!( - "{}/{}/{}", - self.catalog_root, TABLE_CATALOG_MIGRATION_ROOT, TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_LOCK - ) - } - - fn table_bucket_root_prefix(&self, table_bucket: &str) -> String { - format!("{}/{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT, table_catalog_path_hash(table_bucket)) - } -} - -#[derive(Clone)] -pub(crate) struct ObjectTableCatalogStore { - backend: B, - paths: TableCatalogObjectPaths, -} - -fn table_catalog_backing_manifest( - paths: &TableCatalogObjectPaths, - namespace: &Namespace, - table: &IdentifierSegment, - entry: &TableEntry, - commit_recovery: &TableCommitRecoveryReport, -) -> TableCatalogBackingManifest { - let recovery_required = commit_recovery.staged_before_table_update_count > 0 - || commit_recovery.finalization_required_count > 0 - || commit_recovery.idempotency_repair_required_count > 0; - let manual_review_required = commit_recovery.manual_review_count > 0; - let wal_status = if manual_review_required { - TableCatalogWalStatus::ManualReviewRequired - } else if recovery_required { - TableCatalogWalStatus::RecoveryRequired - } else { - TableCatalogWalStatus::Recoverable - }; - let migration_status = if manual_review_required { - TableCatalogBackingMigrationStatus::ManualReviewRequired - } else if recovery_required { - TableCatalogBackingMigrationStatus::RecoveryRequired - } else { - TableCatalogBackingMigrationStatus::ReadyToSnapshot - }; - let mut blockers = Vec::new(); - if recovery_required { - blockers.push(TableCatalogBackingMigrationBlocker::CommitRecoveryRequired); - } - if manual_review_required { - blockers.push(TableCatalogBackingMigrationBlocker::CommitManualReviewRequired); - } - - TableCatalogBackingManifest { - version: TABLE_CATALOG_BACKING_MANIFEST_VERSION, - current: TableCatalogBackingProfile { - kind: TableCatalogBackingKind::ObjectBacked, - authority: TableCatalogAuthority::RustfsSysObject, - consistency: TableCatalogConsistencyMode::ConditionalObjectCas, - durability: TableCatalogDurabilityMode::StagedCommitLogBeforePointerUpdate, - current_pointer_path: paths.table_entry_path(&entry.table_bucket, namespace, table), - wal: TableCatalogWalState { - status: wal_status, - commit_log_prefix: paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id), - idempotency_index_prefix: paths.commit_idempotency_entries_prefix(&entry.table_bucket, &entry.table_id), - committed_generation: entry.generation, - staged_before_table_update_count: commit_recovery.staged_before_table_update_count, - finalization_required_count: commit_recovery.finalization_required_count, - idempotency_repair_required_count: commit_recovery.idempotency_repair_required_count, - manual_review_count: commit_recovery.manual_review_count, - }, - snapshot: TableCatalogSnapshotState { - export_api: "GET /iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export".to_string(), - includes_table_bucket: true, - includes_namespace: true, - includes_table_pointer: true, - includes_backing_manifest: true, - }, - }, - migration: TableCatalogBackingMigrationPlan { - source_kind: TableCatalogBackingKind::ObjectBacked, - target_kind: TableCatalogBackingKind::StrongKvWal, - status: migration_status, - required_steps: vec![ - TableCatalogBackingMigrationStep::SnapshotCatalogExport, - TableCatalogBackingMigrationStep::ReplayCommitLog, - TableCatalogBackingMigrationStep::VerifyCurrentPointer, - TableCatalogBackingMigrationStep::EnableSingleWriterFencing, - TableCatalogBackingMigrationStep::CutOverLinearizableReads, - ], - blockers, - }, - ha: TableCatalogHaPolicy { - writer_region_model: TableCatalogHaWriterModel::SingleActiveWriterRegion, - read_replica_strategy: TableCatalogReadReplicaStrategy::ReadOnlyReplicasForListAndLoad, - commit_read_requirement: TableCatalogCommitReadRequirement::LinearizableLeaderRead, - active_active_supported: false, - failover_requires_operator_promotion: true, - }, - scale_validation: TableCatalogScaleValidation { - status: TableCatalogScaleValidationStatus::MatrixPublished, - benchmark_required: true, - required_scenarios: vec![ - TableCatalogScaleValidationScenario::ConcurrentCommitCas, - TableCatalogScaleValidationScenario::CommitLogRecoveryReplay, - TableCatalogScaleValidationScenario::MigrationSnapshotReplay, - TableCatalogScaleValidationScenario::ReadReplicaStaleReadGuard, - TableCatalogScaleValidationScenario::ClientConformanceMatrix, - ], - }, - } -} - -type StrongNamespaceKey = (String, String); -type StrongResourceKey = (String, String, String); -type StrongCommitKey = (String, String, String); -type StrongWarehouseIndex = BTreeMap>; - -#[derive(Clone, Default)] -struct StrongTableCatalogState { - hydrated: bool, - snapshot_etag: Option, - table_buckets: BTreeMap, - namespaces: BTreeMap, - tables: BTreeMap, - views: BTreeMap, - commits: BTreeMap, - idempotency: BTreeMap, - warehouse_index: StrongWarehouseIndex, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -struct StrongCommitSnapshotRecord { - table_bucket: String, - table_id: String, - lookup_key: String, - commit: CommitLogEntry, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -struct StrongTableCatalogSnapshot { - version: u16, - table_buckets: Vec, - namespaces: Vec, - tables: Vec, - views: Vec, - commits: Vec, - idempotency: Vec, -} - -#[derive(Debug, Clone, PartialEq, Serialize)] -struct StrongTableCatalogBucketSnapshot { - table_bucket: TableBucketEntry, - namespaces: Vec, - tables: Vec, - views: Vec, - commits: Vec, - idempotency: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -enum TableCatalogBackingMigrationFenceStatus { - Preparing, - Materialized, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct TableCatalogBackingMigrationGlobalFence { - version: u16, - migration_id: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct TableCatalogBackingMigrationFence { - version: u16, - table_bucket: String, - migration_id: String, - status: TableCatalogBackingMigrationFenceStatus, - target_bucket_existed: bool, - source_fingerprint: Option, - target_snapshot_etag: Option, -} - -fn table_catalog_bucket_snapshot_fingerprint(snapshot: &StrongTableCatalogBucketSnapshot) -> TableCatalogStoreResult { - let data = serde_json::to_vec(snapshot) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode catalog migration snapshot: {err}")))?; - Ok(hex_simd::encode_to_string(Sha256::digest(data), hex_simd::AsciiCase::Lower)) -} - -#[derive(Clone)] -pub(crate) struct StrongTableCatalogStore { - object_backend: B, - // Single mutex protecting all catalog state (table_buckets, namespaces, tables, views, commits, idempotency). - // This is intentional: many operations require atomic read-modify-write across multiple fields. - // Splitting into per-field locks would introduce deadlock risk and complexity. - // If lock contention becomes a bottleneck (acquisition time > 10ms), consider: - // 1. Using RwLock for read-heavy paths (but most paths need write access) - // 2. Splitting into logical groups (e.g., metadata vs commits) - // 3. Using optimistic concurrency with version checks - state: Arc>, - // Serializes local snapshot mutations; object ETags fence independent store instances. - write_lock: Arc>, -} - -impl StrongTableCatalogStore -where - B: TableCatalogObjectBackend, -{ - pub fn new(object_backend: B) -> Self { - Self { - object_backend, - state: Arc::new(tokio::sync::Mutex::new(StrongTableCatalogState::default())), - write_lock: Arc::new(tokio::sync::Mutex::new(())), - } - } - - fn namespace_key(table_bucket: &str, namespace: &Namespace) -> StrongNamespaceKey { - (table_bucket.to_string(), namespace.public_name()) - } - - fn table_key(table_bucket: &str, namespace: &Namespace, table: &IdentifierSegment) -> StrongResourceKey { - (table_bucket.to_string(), namespace.public_name(), table.as_str().to_string()) - } - - fn commit_key(table_bucket: &str, table_id: &str, commit_id: &str) -> StrongCommitKey { - (table_bucket.to_string(), table_id.to_string(), commit_id.to_string()) - } - - fn idempotency_key(table_bucket: &str, table_id: &str, idempotency_key: &str) -> StrongCommitKey { - (table_bucket.to_string(), table_id.to_string(), idempotency_key.to_string()) - } - - fn snapshot_object_path() -> String { - format!("{INTERNAL_CATALOG_ROOT}/{STRONG_TABLE_CATALOG_BACKING_ROOT}/{STRONG_TABLE_CATALOG_SNAPSHOT_FILE}") - } - - fn snapshot_from_state_locked(state: &StrongTableCatalogState) -> StrongTableCatalogSnapshot { - StrongTableCatalogSnapshot { - version: STRONG_TABLE_CATALOG_SNAPSHOT_VERSION, - table_buckets: state.table_buckets.values().cloned().collect(), - namespaces: state.namespaces.values().cloned().collect(), - tables: state.tables.values().cloned().collect(), - views: state.views.values().cloned().collect(), - commits: state - .commits - .iter() - .map(|((table_bucket, table_id, lookup_key), commit)| StrongCommitSnapshotRecord { - table_bucket: table_bucket.clone(), - table_id: table_id.clone(), - lookup_key: lookup_key.clone(), - commit: commit.clone(), - }) - .collect(), - idempotency: state - .idempotency - .iter() - .map(|((table_bucket, table_id, lookup_key), commit)| StrongCommitSnapshotRecord { - table_bucket: table_bucket.clone(), - table_id: table_id.clone(), - lookup_key: lookup_key.clone(), - commit: commit.clone(), - }) - .collect(), - } - } - - fn bucket_snapshot_from_state_locked( - state: &StrongTableCatalogState, - table_bucket: &str, - ) -> Option { - let table_bucket_entry = state.table_buckets.get(table_bucket)?.clone(); - Some(StrongTableCatalogBucketSnapshot { - table_bucket: table_bucket_entry, - namespaces: state - .namespaces - .iter() - .filter(|((entry_bucket, _), _)| entry_bucket == table_bucket) - .map(|(_, entry)| entry.clone()) - .collect(), - tables: state - .tables - .iter() - .filter(|((entry_bucket, _, _), _)| entry_bucket == table_bucket) - .map(|(_, entry)| entry.clone()) - .collect(), - views: state - .views - .iter() - .filter(|((entry_bucket, _, _), _)| entry_bucket == table_bucket) - .map(|(_, entry)| entry.clone()) - .collect(), - commits: state - .commits - .iter() - .filter(|((entry_bucket, _, _), _)| entry_bucket == table_bucket) - .map(|((entry_bucket, table_id, lookup_key), commit)| StrongCommitSnapshotRecord { - table_bucket: entry_bucket.clone(), - table_id: table_id.clone(), - lookup_key: lookup_key.clone(), - commit: commit.clone(), - }) - .collect(), - idempotency: state - .idempotency - .iter() - .filter(|((entry_bucket, _, _), _)| entry_bucket == table_bucket) - .map(|((entry_bucket, table_id, lookup_key), commit)| StrongCommitSnapshotRecord { - table_bucket: entry_bucket.clone(), - table_id: table_id.clone(), - lookup_key: lookup_key.clone(), - commit: commit.clone(), - }) - .collect(), - }) - } - - fn remove_bucket_from_state_locked(state: &mut StrongTableCatalogState, table_bucket: &str) { - state.table_buckets.remove(table_bucket); - state.namespaces.retain(|(entry_bucket, _), _| entry_bucket != table_bucket); - state.tables.retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); - state.views.retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); - state.commits.retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); - state - .idempotency - .retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); - state.warehouse_index.remove(table_bucket); - } - - fn insert_bucket_snapshot_locked( - state: &mut StrongTableCatalogState, - snapshot: StrongTableCatalogBucketSnapshot, - ) -> TableCatalogStoreResult<()> { - let table_bucket = snapshot.table_bucket.table_bucket.clone(); - Self::remove_bucket_from_state_locked(state, &table_bucket); - state.table_buckets.insert(table_bucket.clone(), snapshot.table_bucket); - for entry in snapshot.namespaces { - let namespace = parse_namespace_for_store(&entry.namespace)?; - state.namespaces.insert(Self::namespace_key(&table_bucket, &namespace), entry); - } - for entry in snapshot.tables { - let namespace = parse_namespace_for_store(&entry.namespace)?; - let table = parse_table_for_store(&entry.table)?; - state.tables.insert(Self::table_key(&table_bucket, &namespace, &table), entry); - } - for entry in snapshot.views { - let namespace = parse_namespace_for_store(&entry.namespace)?; - let view = parse_table_for_store(&entry.view)?; - state.views.insert(Self::table_key(&table_bucket, &namespace, &view), entry); - } - for record in snapshot.commits { - state.commits.insert( - Self::commit_key(&record.table_bucket, &record.table_id, &record.lookup_key), - record.commit, - ); - } - for record in snapshot.idempotency { - state.idempotency.insert( - Self::idempotency_key(&record.table_bucket, &record.table_id, &record.lookup_key), - record.commit, - ); - } - Self::rebuild_warehouse_index_locked(state) - } - - fn rebuild_warehouse_index_locked(state: &mut StrongTableCatalogState) -> TableCatalogStoreResult<()> { - let mut warehouse_index: StrongWarehouseIndex = BTreeMap::new(); - for ((table_bucket, namespace, table), entry) in &state.tables { - if entry.state != TableCatalogEntryState::Active { - continue; - } - if !state - .table_buckets - .get(table_bucket) - .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) - { - continue; - } - if !state - .namespaces - .get(&(table_bucket.clone(), namespace.clone())) - .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) - { - continue; - } - let Ok(warehouse_object_prefix) = table_warehouse_object_prefix(entry) else { - continue; - }; - let table_key = (table_bucket.clone(), namespace.clone(), table.clone()); - if let Some(existing_key) = warehouse_index - .entry(table_bucket.clone()) - .or_default() - .insert(warehouse_object_prefix.clone(), table_key.clone()) - { - return Err(TableCatalogStoreError::Invalid(format!( - "duplicate active table warehouse location in strong catalog snapshot: {warehouse_object_prefix} is owned by {}/{}/{} and {}/{}/{}", - existing_key.0, existing_key.1, existing_key.2, table_key.0, table_key.1, table_key.2 - ))); - } - } - state.warehouse_index = warehouse_index; - Ok(()) - } - - fn snapshot_from_mutated_state_locked( - state: &mut StrongTableCatalogState, - ) -> TableCatalogStoreResult { - Self::rebuild_warehouse_index_locked(state)?; - Ok(Self::snapshot_from_state_locked(state)) - } - - fn state_from_snapshot( - snapshot: StrongTableCatalogSnapshot, - snapshot_etag: Option, - ) -> TableCatalogStoreResult { - if snapshot.version != STRONG_TABLE_CATALOG_SNAPSHOT_VERSION { - return Err(TableCatalogStoreError::Invalid(format!( - "unsupported strong catalog snapshot version: {}", - snapshot.version - ))); - } - - let mut state = StrongTableCatalogState { - hydrated: true, - snapshot_etag, - ..StrongTableCatalogState::default() - }; - for entry in snapshot.table_buckets { - state.table_buckets.insert(entry.table_bucket.clone(), entry); - } - for entry in snapshot.namespaces { - let namespace = parse_namespace_for_store(&entry.namespace)?; - state - .namespaces - .insert(Self::namespace_key(&entry.table_bucket, &namespace), entry); - } - for entry in snapshot.tables { - let namespace = parse_namespace_for_store(&entry.namespace)?; - let table = parse_table_for_store(&entry.table)?; - state - .tables - .insert(Self::table_key(&entry.table_bucket, &namespace, &table), entry); - } - for entry in snapshot.views { - let namespace = parse_namespace_for_store(&entry.namespace)?; - let view = parse_table_for_store(&entry.view)?; - state - .views - .insert(Self::table_key(&entry.table_bucket, &namespace, &view), entry); - } - for record in snapshot.commits { - state.commits.insert( - Self::commit_key(&record.table_bucket, &record.table_id, &record.lookup_key), - record.commit, - ); - } - for record in snapshot.idempotency { - state.idempotency.insert( - Self::idempotency_key(&record.table_bucket, &record.table_id, &record.lookup_key), - record.commit, - ); - } - Self::rebuild_warehouse_index_locked(&mut state)?; - Ok(state) - } - - fn snapshot_write_precondition_locked(state: &StrongTableCatalogState) -> TableCatalogPutPrecondition { - state - .snapshot_etag - .as_ref() - .map_or(TableCatalogPutPrecondition::IfAbsent, |etag| { - TableCatalogPutPrecondition::IfMatch(etag.clone()) - }) - } - - fn snapshot_draft_context_locked(state: &StrongTableCatalogState) -> (TableCatalogPutPrecondition, StrongTableCatalogState) { - (Self::snapshot_write_precondition_locked(state), state.clone()) - } - - async fn hydrate_state(&self) -> TableCatalogStoreResult<()> { - let Some(current_snapshot_etag) = ({ - let state = self.state.lock().await; - if state.hydrated { - Some(state.snapshot_etag.clone()) - } else { - None - } - }) else { - return self.reload_state_from_durable().await; - }; - - let snapshot_metadata = self - .object_backend - .object_metadata(RUSTFS_META_BUCKET, &Self::snapshot_object_path()) - .await?; - match (snapshot_metadata, current_snapshot_etag.as_deref()) { - (None, None) => Ok(()), - (Some(metadata), Some(current_etag)) if metadata.etag.as_deref() == Some(current_etag) => Ok(()), - _ => self.reload_state_from_durable().await, - } - } - - async fn reload_state_from_durable(&self) -> TableCatalogStoreResult<()> { - let snapshot_object = self - .object_backend - .read_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path()) - .await?; - let mut state = self.state.lock().await; - if let Some(snapshot_object) = snapshot_object { - let snapshot = serde_json::from_slice::(&snapshot_object.data) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to decode strong catalog snapshot: {err}")))?; - *state = Self::state_from_snapshot(snapshot, snapshot_object.etag)?; - } else { - *state = StrongTableCatalogState { - hydrated: true, - ..StrongTableCatalogState::default() - }; - } - Ok(()) - } - - async fn persist_snapshot( - &self, - snapshot: StrongTableCatalogSnapshot, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> { - let data = serde_json::to_vec(&snapshot) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode strong catalog snapshot: {err}")))?; - self.object_backend - .put_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), data, precondition) - .await - } - - async fn finalize_snapshot_write( - &self, - snapshot: StrongTableCatalogSnapshot, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> { - match self.persist_snapshot(snapshot, precondition).await { - Ok(()) => self.reload_state_from_durable().await, - Err(err) => { - let _ = self.reload_state_from_durable().await; - Err(err) - } - } - } - - async fn materialize_bucket_snapshot( - &self, - source: StrongTableCatalogBucketSnapshot, - ) -> TableCatalogStoreResult<(String, bool)> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - let table_bucket = source.table_bucket.table_bucket.clone(); - let (snapshot, precondition) = { - let state = self.state.lock().await; - if let Some(current) = Self::bucket_snapshot_from_state_locked(&state, &table_bucket) { - if current != source { - return Err(TableCatalogStoreError::Conflict(format!( - "durable strong catalog already contains different state for table bucket {table_bucket}" - ))); - } - let snapshot_etag = state - .snapshot_etag - .clone() - .ok_or_else(|| TableCatalogStoreError::Internal("durable strong catalog snapshot has no etag".to_string()))?; - return Ok((snapshot_etag, false)); - } - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - Self::insert_bucket_snapshot_locked(&mut draft_state, source)?; - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await?; - let state = self.state.lock().await; - let snapshot_etag = state - .snapshot_etag - .clone() - .ok_or_else(|| TableCatalogStoreError::Internal("durable strong catalog snapshot has no etag".to_string()))?; - Ok((snapshot_etag, true)) - } - - async fn remove_bucket_snapshot_if_unchanged( - &self, - table_bucket: &str, - expected_fingerprint: &str, - ) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - let (snapshot, precondition) = { - let state = self.state.lock().await; - let Some(current) = Self::bucket_snapshot_from_state_locked(&state, table_bucket) else { - return Ok(()); - }; - if table_catalog_bucket_snapshot_fingerprint(¤t)? != expected_fingerprint { - return Err(TableCatalogStoreError::Conflict(format!( - "durable strong catalog state changed after materializing table bucket {table_bucket}" - ))); - } - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - Self::remove_bucket_from_state_locked(&mut draft_state, table_bucket); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await - } - - async fn bucket_snapshot_fingerprint(&self, table_bucket: &str) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let state = self.state.lock().await; - Self::bucket_snapshot_from_state_locked(&state, table_bucket) - .as_ref() - .map(table_catalog_bucket_snapshot_fingerprint) - .transpose() - } - - fn require_table_bucket_in_state(state: &StrongTableCatalogState, table_bucket: &str) -> TableCatalogStoreResult<()> { - if !state.table_buckets.contains_key(table_bucket) { - return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); - } - Ok(()) - } - - fn ensure_table_warehouse_prefix_available_locked( - state: &StrongTableCatalogState, - candidate: &TableEntry, - candidate_key: &StrongResourceKey, - ) -> TableCatalogStoreResult<()> { - if candidate.state != TableCatalogEntryState::Active { - return Ok(()); - } - let candidate_prefix = table_warehouse_object_prefix(candidate)?; - for (existing_key, existing) in &state.tables { - if existing_key == candidate_key - || existing.table_bucket != candidate.table_bucket - || existing.state != TableCatalogEntryState::Active - { - continue; - } - let Ok(existing_prefix) = table_warehouse_object_prefix(existing) else { - continue; - }; - if existing_prefix == candidate_prefix { - return Err(TableCatalogStoreError::Conflict(format!( - "table warehouse location is already registered: {candidate_prefix}" - ))); - } - } - Ok(()) - } - - fn table_commit_recovery_report_for_entry_locked( - state: &StrongTableCatalogState, - entry: &TableEntry, - ) -> TableCommitRecoveryReport { - let mut commits = state - .commits - .iter() - .filter(|((table_bucket, table_id, _), _)| table_bucket == &entry.table_bucket && table_id == &entry.table_id) - .map(|((_, _, _), commit_log)| { - let idempotency_commit = commit_log.idempotency_key.as_deref().and_then(|idempotency_key| { - state - .idempotency - .get(&Self::idempotency_key(&entry.table_bucket, &entry.table_id, idempotency_key)) - }); - table_commit_recovery_entry(entry, commit_log, idempotency_commit) - }) - .collect::>(); - commits.sort_by(|left, right| left.commit_id.cmp(&right.commit_id)); - - let finalization_required_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::FinalizationRequired)) - .count(); - let idempotency_repair_required_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::IdempotencyIndexRepairRequired)) - .count(); - let staged_before_table_update_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::StagedBeforeTableUpdate)) - .count(); - let manual_review_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::ManualReview)) - .count(); - let finalized_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::Committed)) - .count(); - - TableCommitRecoveryReport { - table_bucket: entry.table_bucket.clone(), - namespace: entry.namespace.clone(), - table: entry.table.clone(), - table_id: entry.table_id.clone(), - current_metadata_location: entry.metadata_location.clone(), - current_version_token: entry.version_token.clone(), - current_generation: entry.generation, - commits, - staged_before_table_update_count, - finalization_required_count, - idempotency_repair_required_count, - manual_review_count, - finalized_count, - } - } - - fn validate_new_table_commit_locked( - state: &StrongTableCatalogState, - key: &StrongResourceKey, - request: &TableCommitRequest, - namespace: &Namespace, - table: &IdentifierSegment, - ) -> TableCatalogStoreResult { - let Some(current) = state.tables.get(key).cloned() else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - request.table_bucket, request.namespace, request.table - ))); - }; - let commit_key = Self::commit_key(&request.table_bucket, ¤t.table_id, &request.commit_id); - let existing_commit = state.commits.get(&commit_key); - let idempotency_key = request - .idempotency_key - .as_deref() - .map(|idempotency_key| Self::idempotency_key(&request.table_bucket, ¤t.table_id, idempotency_key)); - let existing_idempotency_commit = idempotency_key.as_ref().and_then(|key| state.idempotency.get(key)); - - if let Some(existing) = existing_commit { - if !commit_log_matches_request(existing, request, ¤t.table_id) { - return Err(TableCatalogStoreError::Conflict(format!( - "commit id already exists: {}", - request.commit_id - ))); - } - if matches!(existing.status, CommitLogStatus::Committed) || table_matches_committed_log(¤t, existing) { - return Ok(current); - } - return Err(TableCatalogStoreError::Conflict( - "existing commit record does not match current table state".to_string(), - )); - } - if let Some(existing) = existing_idempotency_commit - && !commit_log_matches_request(existing, request, ¤t.table_id) - { - return Err(TableCatalogStoreError::Conflict("idempotency key already exists".to_string())); - } - if existing_idempotency_commit.is_some() { - return Err(TableCatalogStoreError::Conflict( - "idempotency key exists without a recoverable commit record".to_string(), - )); - } - if current.version_token != request.expected_version_token { - return Err(TableCatalogStoreError::Conflict( - "current table version token does not match expected token".to_string(), - )); - } - if current.metadata_location != request.expected_metadata_location { - return Err(TableCatalogStoreError::Conflict( - "current table metadata location does not match expected location".to_string(), - )); - } - if !is_valid_table_metadata_location(namespace, table, &request.new_metadata_location) { - return Err(TableCatalogStoreError::Invalid( - "new metadata location must be inside the table metadata directory".to_string(), - )); - } - Ok(current) - } - - fn committed_existing_result_locked( - state: &mut StrongTableCatalogState, - request: &TableCommitRequest, - current: TableEntry, - ) -> Option { - let commit_key = Self::commit_key(&request.table_bucket, ¤t.table_id, &request.commit_id); - let existing = state.commits.get(&commit_key)?; - if !commit_log_matches_request(existing, request, ¤t.table_id) { - return None; - } - if !matches!(existing.status, CommitLogStatus::Committed) && !table_matches_committed_log(¤t, existing) { - return None; - } - - let mut committed = existing.clone(); - committed.status = CommitLogStatus::Committed; - state.commits.insert(commit_key, committed.clone()); - if let Some(idempotency_key) = committed.idempotency_key.as_deref() { - state.idempotency.insert( - Self::idempotency_key(&request.table_bucket, ¤t.table_id, idempotency_key), - committed.clone(), - ); - } - Some(TableCommitResult { - table: current, - commit_log: committed, - }) - } - - fn apply_commit_locked( - state: &mut StrongTableCatalogState, - request: &TableCommitRequest, - namespace: &Namespace, - table: &IdentifierSegment, - next_warehouse_location: Option, - ) -> TableCatalogStoreResult { - let key = Self::table_key(&request.table_bucket, namespace, table); - let current = Self::validate_new_table_commit_locked(state, &key, request, namespace, table)?; - if let Some(result) = Self::committed_existing_result_locked(state, request, current.clone()) { - return Ok(result); - } - - let commit_log = CommitLogEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - commit_id: request.commit_id.clone(), - idempotency_key: request.idempotency_key.clone(), - table_id: current.table_id.clone(), - operation: request.operation.clone(), - expected_version_token: request.expected_version_token.clone(), - new_version_token: format!("token-{}", Uuid::new_v4()), - previous_metadata_location: current.metadata_location.clone(), - new_metadata_location: request.new_metadata_location.clone(), - requirements: request.requirements.clone(), - status: CommitLogStatus::Committed, - writer: request.writer.clone(), - created_at: None, - updated_at: None, - }; - - let mut next = current; - next.metadata_location = commit_log.new_metadata_location.clone(); - if let Some(warehouse_location) = next_warehouse_location { - next.warehouse_location = warehouse_location; - } - Self::ensure_table_warehouse_prefix_available_locked(state, &next, &key)?; - next.version_token = commit_log.new_version_token.clone(); - next.generation = next.generation.saturating_add(1); - - let commit_key = Self::commit_key(&request.table_bucket, &next.table_id, &request.commit_id); - state.commits.insert(commit_key, commit_log.clone()); - if let Some(idempotency_key) = request.idempotency_key.as_deref() { - state.idempotency.insert( - Self::idempotency_key(&request.table_bucket, &next.table_id, idempotency_key), - commit_log.clone(), - ); - } - state.tables.insert(key, next.clone()); - - Ok(TableCommitResult { table: next, commit_log }) - } - - pub(crate) async fn plan_table_commit_recovery( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let key = Self::table_key(table_bucket, &namespace, &table); - let state = self.state.lock().await; - let Some(entry) = state.tables.get(&key) else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - Ok(Self::table_commit_recovery_report_for_entry_locked(&state, entry)) - } -} - -#[derive(Clone)] -pub(crate) enum ConfiguredTableCatalogStore -where - B: TableCatalogObjectBackend, -{ - ObjectBacked(ObjectTableCatalogStore), - DurableStrong(StrongTableCatalogStore), -} - -impl ConfiguredTableCatalogStore -where - B: TableCatalogObjectBackend, -{ - pub(crate) fn from_env(backend: B) -> TableCatalogStoreResult { - Ok(Self::new(backend, TableCatalogBackingMode::from_env()?)) - } - - pub(crate) fn new(backend: B, mode: TableCatalogBackingMode) -> Self { - match mode { - TableCatalogBackingMode::ObjectBacked => Self::ObjectBacked(ObjectTableCatalogStore::new(backend)), - TableCatalogBackingMode::DurableStrong => Self::DurableStrong(StrongTableCatalogStore::new(backend)), - } - } - - pub(crate) fn backing_mode(&self) -> TableCatalogBackingMode { - match self { - Self::ObjectBacked(_) => TableCatalogBackingMode::ObjectBacked, - Self::DurableStrong(_) => TableCatalogBackingMode::DurableStrong, - } - } - - fn unsupported_for_durable_strong(operation: &str) -> TableCatalogStoreError { - TableCatalogStoreError::Invalid(format!( - "{operation} is not supported with {TABLE_CATALOG_BACKING_DURABLE_STRONG} table catalog backing" - )) - } -} - -#[async_trait::async_trait] -impl TableCatalogStore for StrongTableCatalogStore -where - B: TableCatalogObjectBackend, -{ - async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let state = self.state.lock().await; - Ok(state.table_buckets.get(table_bucket).cloned()) - } - - async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - validate_catalog_entry_version("table bucket", entry.version)?; - if entry.table_bucket.is_empty() { - return Err(TableCatalogStoreError::Invalid("table bucket name cannot be empty".to_string())); - } - if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE { - return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string())); - } - - let (snapshot, precondition) = { - let state = self.state.lock().await; - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.table_buckets.insert(entry.table_bucket.clone(), entry); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await - } - - async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - validate_catalog_entry_version("namespace", entry.version)?; - let namespace = parse_namespace_for_store(&entry.namespace)?; - let key = Self::namespace_key(&entry.table_bucket, &namespace); - let (snapshot, precondition) = { - let state = self.state.lock().await; - Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; - if state.namespaces.contains_key(&key) { - return Err(TableCatalogStoreError::Conflict(format!( - "catalog object already exists: namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.namespaces.insert(key, entry); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await - } - - async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let state = self.state.lock().await; - let mut entries = state - .namespaces - .iter() - .filter(|((bucket, _), _)| bucket == table_bucket) - .map(|(_, entry)| entry.clone()) - .collect::>(); - entries.sort_by(|left, right| left.namespace.cmp(&right.namespace)); - Ok(entries) - } - - async fn list_namespaces_page( - &self, - table_bucket: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let cursor = catalog_list_cursor(cursor, STRONG_CATALOG_LIST_CURSOR_PREFIX)?; - let cursor = cursor - .map(parse_namespace_for_store) - .transpose()? - .map(|namespace| namespace.public_name()); - let start = match cursor { - Some(cursor) => Bound::Excluded((table_bucket.to_string(), cursor)), - None => Bound::Included((table_bucket.to_string(), String::new())), - }; - let state = self.state.lock().await; - let entries = state - .namespaces - .range((start, Bound::Unbounded)) - .take_while(|((bucket, _), _)| bucket == table_bucket) - .take(limit.get().saturating_add(1)) - .map(|(_, entry)| entry.clone()) - .collect(); - Ok(finish_catalog_list_page(entries, limit, STRONG_CATALOG_LIST_CURSOR_PREFIX, |entry| { - &entry.namespace - })) - } - - async fn get_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let state = self.state.lock().await; - Ok(state.namespaces.get(&Self::namespace_key(table_bucket, &namespace)).cloned()) - } - - async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let key = Self::namespace_key(table_bucket, &namespace); - let (snapshot, precondition) = { - let state = self.state.lock().await; - if !state.namespaces.contains_key(&key) { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - table_bucket, - namespace.public_name() - ))); - } - if state - .tables - .keys() - .any(|(bucket, namespace_name, _)| bucket == table_bucket && namespace_name == &namespace.public_name()) - || state - .views - .keys() - .any(|(bucket, namespace_name, _)| bucket == table_bucket && namespace_name == &namespace.public_name()) - { - return Err(TableCatalogStoreError::Conflict(format!( - "namespace {}/{} is not empty", - table_bucket, - namespace.public_name() - ))); - } - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.namespaces.remove(&key); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await - } - - async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { - self.register_table(entry).await - } - - async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - validate_catalog_entry_version("table", entry.version)?; - let namespace = parse_namespace_for_store(&entry.namespace)?; - let table = parse_table_for_store(&entry.table)?; - table_warehouse_object_prefix(&entry)?; - let key = Self::table_key(&entry.table_bucket, &namespace, &table); - let (snapshot, precondition) = { - let state = self.state.lock().await; - Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; - if !state - .namespaces - .contains_key(&Self::namespace_key(&entry.table_bucket, &namespace)) - { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - if state.tables.contains_key(&key) { - return Err(TableCatalogStoreError::Conflict(format!( - "catalog object already exists: table {}/{}/{}", - entry.table_bucket, entry.namespace, entry.table - ))); - } - Self::ensure_table_warehouse_prefix_available_locked(&state, &entry, &key)?; - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.tables.insert(key, entry); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await - } - - async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let state = self.state.lock().await; - let mut entries = state - .tables - .iter() - .filter(|((bucket, namespace_name, _), _)| bucket == table_bucket && namespace_name == &namespace.public_name()) - .map(|(_, entry)| entry.clone()) - .collect::>(); - entries.sort_by(|left, right| left.table.cmp(&right.table)); - Ok(entries) - } - - async fn list_tables_page( - &self, - table_bucket: &str, - namespace: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?.public_name(); - let cursor = catalog_list_cursor(cursor, STRONG_CATALOG_LIST_CURSOR_PREFIX)?; - let cursor = cursor - .map(parse_table_for_store) - .transpose()? - .map(|table| table.as_str().to_string()); - let start = match cursor { - Some(cursor) => Bound::Excluded((table_bucket.to_string(), namespace.clone(), cursor)), - None => Bound::Included((table_bucket.to_string(), namespace.clone(), String::new())), - }; - let state = self.state.lock().await; - let entries = state - .tables - .range((start, Bound::Unbounded)) - .take_while(|((bucket, entry_namespace, _), _)| bucket == table_bucket && entry_namespace == &namespace) - .take(limit.get().saturating_add(1)) - .map(|(_, entry)| entry.clone()) - .collect(); - Ok(finish_catalog_list_page(entries, limit, STRONG_CATALOG_LIST_CURSOR_PREFIX, |entry| { - &entry.table - })) - } - - async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let state = self.state.lock().await; - Ok(state.tables.get(&Self::table_key(table_bucket, &namespace, &table)).cloned()) - } - - async fn resolve_table_data_plane_resource( - &self, - table_bucket: &str, - object: &str, - ) -> TableCatalogStoreResult> { - if table_bucket.is_empty() || object.is_empty() { - return Ok(None); - } - - self.hydrate_state().await?; - let state = self.state.lock().await; - let Some(bucket_entry) = state.table_buckets.get(table_bucket) else { - return Ok(None); - }; - if bucket_entry.state != TableCatalogEntryState::Active { - return Ok(None); - } - - let Some(bucket_index) = state.warehouse_index.get(table_bucket) else { - return Ok(None); - }; - - for warehouse_object_prefix in warehouse_index_candidate_prefixes(object) { - if let Some(table_key) = bucket_index.get(warehouse_object_prefix) { - let Some(table) = state.tables.get(table_key) else { - continue; - }; - return Ok(Some(table_data_plane_resource_from_entry( - table.clone(), - warehouse_object_prefix.to_string(), - ))); - } - } - Ok(None) - } - - async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - let commit_started = Instant::now(); - record_table_commit_attempt(&request.operation); - let namespace = parse_namespace_for_store(&request.namespace)?; - let table = parse_table_for_store(&request.table)?; - let key = Self::table_key(&request.table_bucket, &namespace, &table); - - let committed_existing_result = { - let state = self.state.lock().await; - let current = Self::validate_new_table_commit_locked(&state, &key, &request, &namespace, &table); - match current { - Ok(current) => { - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - Self::committed_existing_result_locked(&mut draft_state, &request, current).map(|result| { - Self::snapshot_from_mutated_state_locked(&mut draft_state) - .map(|snapshot| (result, snapshot, precondition)) - }) - } - Err(error) => { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(error), - ); - } - } - }; - if let Some(prepared_result) = committed_existing_result { - let result = match prepared_result { - Ok((result, snapshot, precondition)) => { - self.finalize_snapshot_write(snapshot, precondition).await.map(|_| result) - } - Err(err) => Err(err), - }; - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - result, - ); - } - - let Some(new_metadata_object) = self - .object_backend - .read_object(&request.table_bucket, &request.new_metadata_location) - .await? - else { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::NotFound(format!( - "new metadata object {}", - request.new_metadata_location - ))), - ); - }; - let next_warehouse_location = - table_metadata_warehouse_location(&request.table_bucket, &request.new_metadata_location, &new_metadata_object)?; - - let cas_started = Instant::now(); - let prepared_result = { - let state = self.state.lock().await; - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - match Self::apply_commit_locked(&mut draft_state, &request, &namespace, &table, next_warehouse_location) { - Ok(result) => { - Self::snapshot_from_mutated_state_locked(&mut draft_state).map(|snapshot| (result, snapshot, precondition)) - } - Err(err) => Err(err), - } - }; - let result = match prepared_result { - Ok((result, snapshot, precondition)) => self.finalize_snapshot_write(snapshot, precondition).await.map(|_| result), - Err(err) => Err(err), - }; - let cas_result = result.as_ref().map(|_| ()).map_err(Clone::clone); - record_table_commit_cas_result(&request.operation, cas_started, &cas_result); - table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - result, - ) - } - - async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let key = Self::table_key(table_bucket, &namespace, &table); - let (snapshot, precondition) = { - let state = self.state.lock().await; - if !state.tables.contains_key(&key) { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - } - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.tables.remove(&key); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await - } - - async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - validate_catalog_entry_version("view", entry.version)?; - validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; - let namespace = parse_namespace_for_store(&entry.namespace)?; - let view = parse_table_for_store(&entry.view)?; - let key = Self::table_key(&entry.table_bucket, &namespace, &view); - let (snapshot, precondition) = { - let state = self.state.lock().await; - Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; - if !state - .namespaces - .contains_key(&Self::namespace_key(&entry.table_bucket, &namespace)) - { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - if state.views.contains_key(&key) { - return Err(TableCatalogStoreError::Conflict(format!( - "catalog object already exists: view {}/{}/{}", - entry.table_bucket, entry.namespace, entry.view - ))); - } - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.views.insert(key, entry); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await - } - - async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let state = self.state.lock().await; - let mut entries = state - .views - .iter() - .filter(|((bucket, namespace_name, _), _)| bucket == table_bucket && namespace_name == &namespace.public_name()) - .map(|(_, entry)| entry.clone()) - .collect::>(); - entries.sort_by(|left, right| left.view.cmp(&right.view)); - Ok(entries) - } - - async fn list_views_page( - &self, - table_bucket: &str, - namespace: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?.public_name(); - let cursor = catalog_list_cursor(cursor, STRONG_CATALOG_LIST_CURSOR_PREFIX)?; - let cursor = cursor - .map(parse_table_for_store) - .transpose()? - .map(|view| view.as_str().to_string()); - let start = match cursor { - Some(cursor) => Bound::Excluded((table_bucket.to_string(), namespace.clone(), cursor)), - None => Bound::Included((table_bucket.to_string(), namespace.clone(), String::new())), - }; - let state = self.state.lock().await; - let entries = state - .views - .range((start, Bound::Unbounded)) - .take_while(|((bucket, entry_namespace, _), _)| bucket == table_bucket && entry_namespace == &namespace) - .take(limit.get().saturating_add(1)) - .map(|(_, entry)| entry.clone()) - .collect(); - Ok(finish_catalog_list_page(entries, limit, STRONG_CATALOG_LIST_CURSOR_PREFIX, |entry| { - &entry.view - })) - } - - async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let view = parse_table_for_store(view)?; - let state = self.state.lock().await; - Ok(state.views.get(&Self::table_key(table_bucket, &namespace, &view)).cloned()) - } - - async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(&request.namespace)?; - let view = parse_table_for_store(&request.view)?; - if !is_valid_view_metadata_location(&namespace, &view, &request.new_metadata_location) { - return Err(TableCatalogStoreError::Invalid( - "new metadata location must be inside the view metadata directory".to_string(), - )); - } - let Some(new_metadata_object) = self - .object_backend - .read_object(&request.table_bucket, &request.new_metadata_location) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!( - "new view metadata object {}", - request.new_metadata_location - ))); - }; - let next_warehouse_location = - view_metadata_warehouse_location(&request.table_bucket, &request.new_metadata_location, &new_metadata_object)?; - - let key = Self::table_key(&request.table_bucket, &namespace, &view); - let (snapshot, precondition, next) = { - let state = self.state.lock().await; - let Some(current) = state.views.get(&key).cloned() else { - return Err(TableCatalogStoreError::NotFound(format!( - "view {}/{}/{}", - request.table_bucket, request.namespace, request.view - ))); - }; - if current.version_token != request.expected_version_token { - return Err(TableCatalogStoreError::Conflict( - "current view version token does not match expected token".to_string(), - )); - } - if current.metadata_location != request.expected_metadata_location { - return Err(TableCatalogStoreError::Conflict( - "current view metadata location does not match expected location".to_string(), - )); - } - - let mut next = current; - next.metadata_location = request.new_metadata_location; - if let Some(warehouse_location) = next_warehouse_location { - next.warehouse_location = warehouse_location; - } - next.version_token = format!("token-{}", Uuid::new_v4()); - next.generation = next.generation.saturating_add(1); - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.views.insert(key, next.clone()); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition, next) - }; - self.finalize_snapshot_write(snapshot, precondition).await?; - Ok(ViewCommitResult { view: next }) - } - - async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; - let namespace = parse_namespace_for_store(namespace)?; - let view = parse_table_for_store(view)?; - let key = Self::table_key(table_bucket, &namespace, &view); - let (snapshot, precondition) = { - let state = self.state.lock().await; - if !state.views.contains_key(&key) { - return Err(TableCatalogStoreError::NotFound(format!( - "view {}/{}/{}", - table_bucket, - namespace.public_name(), - view.as_str() - ))); - } - let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.views.remove(&key); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) - }; - self.finalize_snapshot_write(snapshot, precondition).await - } - - async fn get_commit_by_id( - &self, - table_bucket: &str, - table_id: &str, - commit_id: &str, - ) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let state = self.state.lock().await; - Ok(state - .commits - .get(&Self::commit_key(table_bucket, table_id, commit_id)) - .cloned()) - } - - async fn get_commit_by_idempotency_key( - &self, - table_bucket: &str, - table_id: &str, - idempotency_key: &str, - ) -> TableCatalogStoreResult> { - self.hydrate_state().await?; - let state = self.state.lock().await; - Ok(state - .idempotency - .get(&Self::idempotency_key(table_bucket, table_id, idempotency_key)) - .cloned()) - } -} - -impl ObjectTableCatalogStore -where - B: TableCatalogObjectBackend, -{ - pub fn new(backend: B) -> Self { - Self { - backend, - paths: TableCatalogObjectPaths::default(), - } - } - - fn catalog_bucket(&self) -> &'static str { - RUSTFS_META_BUCKET - } - - async fn list_entry_page( - &self, - prefix: &str, - entry_file: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> - where - T: DeserializeOwned, - { - let cursor = catalog_list_cursor(cursor, OBJECT_CATALOG_LIST_CURSOR_PREFIX)?; - if cursor.is_some_and(|cursor| !cursor.starts_with(prefix)) { - return Err(TableCatalogStoreError::Invalid( - "page cursor does not match this table catalog list operation".to_string(), - )); - } - - let scan_limit = NonZeroUsize::new(TABLE_CATALOG_LIST_MAX_KEYS) - .ok_or_else(|| TableCatalogStoreError::Internal("catalog object scan limit must be positive".to_string()))?; - let mut entries = Vec::with_capacity(limit.get()); - let mut last_entry_path = None; - - let page = self - .backend - .list_objects_page(self.catalog_bucket(), prefix, cursor, scan_limit) - .await?; - let last_scanned_path = page.objects.last().cloned(); - if page.is_truncated && last_scanned_path.is_none() { - return Err(TableCatalogStoreError::Internal("catalog object pagination made no progress".to_string())); - } - if cursor - .zip(last_scanned_path.as_deref()) - .is_some_and(|(cursor, last)| last <= cursor) - { - return Err(TableCatalogStoreError::Internal("catalog object pagination did not advance".to_string())); - } - - for object in page.objects { - if !object.ends_with(entry_file) { - continue; - } - if entries.len() == limit.get() { - let last_entry_path = last_entry_path.ok_or_else(|| { - TableCatalogStoreError::Internal("catalog page cursor is missing its last entry".to_string()) - })?; - return Ok(TableCatalogListPage { - entries, - next_cursor: Some(format!("{OBJECT_CATALOG_LIST_CURSOR_PREFIX}{last_entry_path}")), - }); - } - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { - continue; - }; - last_entry_path = Some(object); - entries.push(entry); - } - - let next_cursor = if page.is_truncated { - last_scanned_path.map(|path| format!("{OBJECT_CATALOG_LIST_CURSOR_PREFIX}{path}")) - } else { - None - }; - Ok(TableCatalogListPage { entries, next_cursor }) - } - - async fn read_backing_migration_fence( - &self, - table_bucket: &str, - ) -> TableCatalogStoreResult)>> { - self.read_entry(self.catalog_bucket(), &self.paths.backing_migration_fence_path(table_bucket)) - .await - } - - async fn acquire_table_bucket_registry_write_permit(&self) -> TableCatalogStoreResult> { - let fence_path = self.paths.backing_migration_global_fence_path(); - let lock_path = self.paths.backing_migration_global_fence_lock_path(); - let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; - if self - .read_entry::(self.catalog_bucket(), &fence_path) - .await? - .is_some() - { - return Err(TableCatalogStoreError::Conflict( - "table bucket registry writes are fenced while durable strong migration is in progress".to_string(), - )); - } - Ok(guard) - } - - async fn acquire_object_backed_catalog_write_permit(&self, table_bucket: &str) -> TableCatalogStoreResult> { - let lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); - let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; - if self.read_backing_migration_fence(table_bucket).await?.is_some() { - return Err(TableCatalogStoreError::Conflict(format!( - "object-backed catalog writes are fenced while table bucket {table_bucket} is prepared for durable strong cutover" - ))); - } - Ok(guard) - } - - async fn ensure_global_backing_migration_fence( - &self, - fence_path: &str, - ) -> TableCatalogStoreResult { - if let Some((fence, _)) = self - .read_entry::(self.catalog_bucket(), fence_path) - .await? - { - if fence.version != TABLE_CATALOG_MIGRATION_VERSION { - return Err(TableCatalogStoreError::Invalid( - "invalid durable strong global migration fence".to_string(), - )); - } - return Ok(fence); - } - let fence = TableCatalogBackingMigrationGlobalFence { - version: TABLE_CATALOG_MIGRATION_VERSION, - migration_id: Uuid::new_v4().to_string(), - }; - self.write_entry(self.catalog_bucket(), fence_path, &fence, TableCatalogPutPrecondition::IfAbsent) - .await?; - Ok(fence) - } - - async fn clear_global_backing_migration_fence_if_unused(&self, fence_path: &str) -> TableCatalogStoreResult<()> { - let bucket_objects = self - .backend - .list_objects(self.catalog_bucket(), &self.paths.table_bucket_entries_prefix()) - .await?; - if bucket_objects - .iter() - .any(|object| object.ends_with(TABLE_CATALOG_MIGRATION_FENCE_FILE)) - { - return Ok(()); - } - self.backend.delete_object(self.catalog_bucket(), fence_path).await - } - - async fn ensure_object_backed_writes_allowed(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { - if self - .backend - .object_exists(self.catalog_bucket(), &self.paths.backing_migration_fence_path(table_bucket)) - .await? - { - return Err(TableCatalogStoreError::Conflict(format!( - "object-backed catalog writes are fenced while table bucket {table_bucket} is prepared for durable strong cutover" - ))); - } - Ok(()) - } - - async fn collect_bucket_snapshot_with_locks( - &self, - table_bucket: &str, - guards: &mut Vec>, - ) -> TableCatalogStoreResult { - let bucket_path = self.paths.table_bucket_entry_path(table_bucket); - guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?); - let Some((table_bucket_entry, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), &bucket_path) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); - }; - if table_bucket_entry.table_bucket != table_bucket { - return Err(TableCatalogStoreError::Invalid(format!( - "table bucket entry does not match migration target {table_bucket}" - ))); - } - - let mut namespaces = Vec::new(); - let mut tables = Vec::new(); - let mut views = Vec::new(); - let mut commits = Vec::new(); - let mut idempotency = Vec::new(); - let namespace_objects = self - .backend - .list_objects(self.catalog_bucket(), &self.paths.namespace_entries_prefix(table_bucket)) - .await?; - let mut unmatched_table_objects = namespace_objects - .iter() - .filter(|object| object.ends_with(TABLE_ENTRY_FILE)) - .cloned() - .collect::>(); - let mut unmatched_view_objects = namespace_objects - .iter() - .filter(|object| object.ends_with(VIEW_ENTRY_FILE)) - .cloned() - .collect::>(); - for namespace_object in namespace_objects - .iter() - .filter(|object| object.ends_with(NAMESPACE_ENTRY_FILE)) - { - guards.push( - self.backend - .acquire_write_lock(self.catalog_bucket(), namespace_object) - .await?, - ); - let Some((namespace_entry, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), namespace_object) - .await? - else { - return Err(TableCatalogStoreError::Conflict(format!( - "namespace changed while preparing durable strong snapshot: {namespace_object}" - ))); - }; - if namespace_entry.table_bucket != table_bucket { - return Err(TableCatalogStoreError::Invalid(format!( - "namespace {} belongs to a different table bucket", - namespace_entry.namespace - ))); - } - let namespace = parse_namespace_for_store(&namespace_entry.namespace)?; - - let table_objects = self - .backend - .list_objects(self.catalog_bucket(), &self.paths.table_entries_prefix(table_bucket, &namespace)) - .await?; - for table_object in table_objects.iter().filter(|object| object.ends_with(TABLE_ENTRY_FILE)) { - unmatched_table_objects.remove(table_object); - guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), table_object).await?); - let Some((table_entry, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), table_object) - .await? - else { - return Err(TableCatalogStoreError::Conflict(format!( - "table changed while preparing durable strong snapshot: {table_object}" - ))); - }; - if table_entry.table_bucket != table_bucket || table_entry.namespace != namespace_entry.namespace { - return Err(TableCatalogStoreError::Invalid(format!( - "table {} does not match its catalog namespace", - table_entry.table - ))); - } - - for commit_object in self - .backend - .list_objects( - self.catalog_bucket(), - &self.paths.commit_log_entries_prefix(table_bucket, &table_entry.table_id), - ) - .await? - .into_iter() - .filter(|object| object.ends_with(".json")) - { - let Some((commit, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), &commit_object) - .await? - else { - return Err(TableCatalogStoreError::Conflict(format!( - "commit log changed while preparing durable strong snapshot: {commit_object}" - ))); - }; - commits.push(StrongCommitSnapshotRecord { - table_bucket: table_bucket.to_string(), - table_id: table_entry.table_id.clone(), - lookup_key: commit.commit_id.clone(), - commit, - }); - } - for idempotency_object in self - .backend - .list_objects( - self.catalog_bucket(), - &self - .paths - .commit_idempotency_entries_prefix(table_bucket, &table_entry.table_id), - ) - .await? - .into_iter() - .filter(|object| object.ends_with(".json")) - { - let Some((commit, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), &idempotency_object) - .await? - else { - return Err(TableCatalogStoreError::Conflict(format!( - "idempotency index changed while preparing durable strong snapshot: {idempotency_object}" - ))); - }; - let lookup_key = commit.idempotency_key.clone().ok_or_else(|| { - TableCatalogStoreError::Invalid(format!("idempotency index {idempotency_object} has no idempotency key")) - })?; - idempotency.push(StrongCommitSnapshotRecord { - table_bucket: table_bucket.to_string(), - table_id: table_entry.table_id.clone(), - lookup_key, - commit, - }); - } - tables.push(table_entry); - } - - let view_objects = self - .backend - .list_objects(self.catalog_bucket(), &self.paths.view_entries_prefix(table_bucket, &namespace)) - .await?; - for view_object in view_objects.iter().filter(|object| object.ends_with(VIEW_ENTRY_FILE)) { - unmatched_view_objects.remove(view_object); - guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), view_object).await?); - let Some((view_entry, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), view_object) - .await? - else { - return Err(TableCatalogStoreError::Conflict(format!( - "view changed while preparing durable strong snapshot: {view_object}" - ))); - }; - if view_entry.table_bucket != table_bucket || view_entry.namespace != namespace_entry.namespace { - return Err(TableCatalogStoreError::Invalid(format!( - "view {} does not match its catalog namespace", - view_entry.view - ))); - } - views.push(view_entry); - } - namespaces.push(namespace_entry); - } - if let Some(object) = unmatched_table_objects.first() { - return Err(TableCatalogStoreError::Invalid(format!( - "table entry has no namespace entry during durable strong migration: {object}" - ))); - } - if let Some(object) = unmatched_view_objects.first() { - return Err(TableCatalogStoreError::Invalid(format!( - "view entry has no namespace entry during durable strong migration: {object}" - ))); - } - - namespaces.sort_by(|left, right| left.namespace.cmp(&right.namespace)); - tables.sort_by(|left, right| (&left.namespace, &left.table).cmp(&(&right.namespace, &right.table))); - views.sort_by(|left, right| (&left.namespace, &left.view).cmp(&(&right.namespace, &right.view))); - commits.sort_by(|left, right| (&left.table_id, &left.lookup_key).cmp(&(&right.table_id, &right.lookup_key))); - idempotency.sort_by(|left, right| (&left.table_id, &left.lookup_key).cmp(&(&right.table_id, &right.lookup_key))); - - let snapshot = StrongTableCatalogBucketSnapshot { - table_bucket: table_bucket_entry, - namespaces, - tables, - views, - commits, - idempotency, - }; - self.validate_bucket_snapshot_for_migration(&snapshot)?; - Ok(snapshot) - } - - fn validate_bucket_snapshot_for_migration(&self, snapshot: &StrongTableCatalogBucketSnapshot) -> TableCatalogStoreResult<()> { - let table_bucket = &snapshot.table_bucket.table_bucket; - let tables_by_id = snapshot - .tables - .iter() - .map(|table| (table.table_id.as_str(), table)) - .collect::>(); - if tables_by_id.len() != snapshot.tables.len() { - return Err(TableCatalogStoreError::Invalid( - "migration snapshot contains duplicate table ids".to_string(), - )); - } - let commits_by_key = snapshot - .commits - .iter() - .map(|record| ((record.table_id.as_str(), record.lookup_key.as_str()), &record.commit)) - .collect::>(); - if commits_by_key.len() != snapshot.commits.len() { - return Err(TableCatalogStoreError::Invalid( - "migration snapshot contains duplicate commit lookup keys".to_string(), - )); - } - let idempotency_by_key = snapshot - .idempotency - .iter() - .map(|record| ((record.table_id.as_str(), record.lookup_key.as_str()), &record.commit)) - .collect::>(); - if idempotency_by_key.len() != snapshot.idempotency.len() { - return Err(TableCatalogStoreError::Invalid( - "migration snapshot contains duplicate idempotency lookup keys".to_string(), - )); - } - for record in &snapshot.commits { - let table = tables_by_id.get(record.table_id.as_str()).ok_or_else(|| { - TableCatalogStoreError::Invalid(format!("commit {} has no table in migration snapshot", record.commit.commit_id)) - })?; - if record.table_bucket != *table_bucket - || record.commit.table_id != record.table_id - || record.lookup_key != record.commit.commit_id - { - return Err(TableCatalogStoreError::Invalid(format!( - "commit {} does not match its migration snapshot owner", - record.commit.commit_id - ))); - } - let indexed = record - .commit - .idempotency_key - .as_deref() - .and_then(|idempotency_key| idempotency_by_key.get(&(record.table_id.as_str(), idempotency_key)).copied()); - let recovery = table_commit_recovery_entry(table, &record.commit, indexed); - if recovery.recovery_state != TableCommitRecoveryState::Committed { - return Err(TableCatalogStoreError::Conflict(format!( - "commit {} requires catalog recovery before durable strong migration", - record.commit.commit_id - ))); - } - } - for record in &snapshot.idempotency { - let _table = tables_by_id.get(record.table_id.as_str()).ok_or_else(|| { - TableCatalogStoreError::Invalid(format!( - "idempotency index {} has no table in migration snapshot", - record.lookup_key - )) - })?; - if record.table_bucket != *table_bucket || record.commit.table_id != record.table_id { - return Err(TableCatalogStoreError::Invalid(format!( - "idempotency index {} does not match its migration snapshot owner", - record.lookup_key - ))); - } - if record.commit.idempotency_key.as_deref() != Some(record.lookup_key.as_str()) { - return Err(TableCatalogStoreError::Invalid(format!( - "idempotency index {} does not match its commit payload", - record.lookup_key - ))); - } - let committed = commits_by_key - .get(&(record.table_id.as_str(), record.commit.commit_id.as_str())) - .ok_or_else(|| { - TableCatalogStoreError::Invalid(format!( - "idempotency index {} has no commit record in migration snapshot", - record.lookup_key - )) - })?; - if *committed != &record.commit { - return Err(TableCatalogStoreError::Conflict(format!( - "idempotency index {} requires catalog recovery before durable strong migration", - record.lookup_key - ))); - } - } - - let mut state = StrongTableCatalogState { - hydrated: true, - ..StrongTableCatalogState::default() - }; - StrongTableCatalogStore::::insert_bucket_snapshot_locked(&mut state, snapshot.clone())?; - if state.namespaces.len() != snapshot.namespaces.len() - || state.tables.len() != snapshot.tables.len() - || state.views.len() != snapshot.views.len() - || state.commits.len() != snapshot.commits.len() - || state.idempotency.len() != snapshot.idempotency.len() - { - return Err(TableCatalogStoreError::Invalid( - "migration snapshot contains duplicate catalog identities".to_string(), - )); - } - Ok(()) - } - - async fn read_entry(&self, bucket: &str, object: &str) -> TableCatalogStoreResult)>> - where - T: DeserializeOwned, - { - self.read_entry_with(bucket, object, |backend, bucket, object| { - Box::pin(async move { backend.read_object(bucket, object).await }) - }) - .await - } - - async fn read_entry_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult)>> - where - T: DeserializeOwned, - { - self.read_entry_with(bucket, object, |backend, bucket, object| { - Box::pin(async move { backend.read_object_unlocked(bucket, object).await }) - }) - .await - } - - async fn read_entry_with<'a, T, F>( - &'a self, - bucket: &'a str, - object: &'a str, - read_object: F, - ) -> TableCatalogStoreResult)>> - where - T: DeserializeOwned, - F: FnOnce( - &'a B, - &'a str, - &'a str, - ) -> std::pin::Pin< - Box>> + Send + 'a>, - >, - { - let Some(object_data) = read_object(&self.backend, bucket, object).await? else { - return Ok(None); - }; - - let entry = serde_json::from_slice(&object_data.data) - .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to parse catalog entry {object}: {err}")))?; - Ok(Some((entry, object_data.etag))) - } - - async fn write_entry( - &self, - bucket: &str, - object: &str, - entry: &T, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> - where - T: Serialize, - { - let data = serde_json::to_vec(entry) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize catalog entry {object}: {err}")))?; - self.backend.put_object(bucket, object, data, precondition).await - } - - async fn write_entry_unlocked( - &self, - bucket: &str, - object: &str, - entry: &T, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> - where - T: Serialize, - { - let data = serde_json::to_vec(entry) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize catalog entry {object}: {err}")))?; - self.backend.put_object_unlocked(bucket, object, data, precondition).await - } - - async fn write_warehouse_index_state_unlocked(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { - let state = TableWarehouseIndexStateEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: table_bucket.to_string(), - state: TableCatalogEntryState::Active, - }; - self.write_entry_unlocked( - self.catalog_bucket(), - &self.paths.warehouse_index_state_path(table_bucket), - &state, - TableCatalogPutPrecondition::Any, - ) - .await - } - - async fn warehouse_index_ready(&self, table_bucket: &str) -> TableCatalogStoreResult { - let Some((state, _)) = self - .read_entry::( - self.catalog_bucket(), - &self.paths.warehouse_index_state_path(table_bucket), - ) - .await? - else { - return Ok(false); - }; - Ok(state.version == TABLE_CATALOG_ENTRY_VERSION - && state.table_bucket == table_bucket - && state.state == TableCatalogEntryState::Active) - } - - async fn warehouse_index_entry_has_active_owner(&self, index: &TableWarehouseIndexEntry) -> TableCatalogStoreResult { - if index.state != TableCatalogEntryState::Active { - return Ok(false); - } - let Some(table) = self.load_table(&index.table_bucket, &index.namespace, &index.table).await? else { - return Ok(false); - }; - if table.state != TableCatalogEntryState::Active { - return Ok(false); - } - let current_prefix = table_warehouse_object_prefix(&table)?; - Ok(current_prefix == index.warehouse_object_prefix) - } - - async fn delete_warehouse_index_object( - &self, - object: &str, - index: &TableWarehouseIndexEntry, - reason: &'static str, - ) -> TableCatalogStoreResult { - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), object).await?; - let Some((current, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), object) - .await? - else { - return Ok(false); - }; - if current != *index { - tracing::warn!( - table_bucket = %index.table_bucket, - namespace = %index.namespace, - table = %index.table, - table_id = %index.table_id, - warehouse_object_prefix = %index.warehouse_object_prefix, - current_namespace = %current.namespace, - current_table = %current.table, - current_table_id = %current.table_id, - reason = %reason, - "skipped deleting table warehouse index because owner changed" - ); - return Ok(false); - } - self.delete_warehouse_index_object_unlocked(object, index, reason).await?; - Ok(true) - } - - async fn delete_warehouse_index_object_unlocked( - &self, - object: &str, - index: &TableWarehouseIndexEntry, - reason: &'static str, - ) -> TableCatalogStoreResult<()> { - self.backend.delete_object_unlocked(self.catalog_bucket(), object).await?; - tracing::warn!( - table_bucket = %index.table_bucket, - namespace = %index.namespace, - table = %index.table, - table_id = %index.table_id, - warehouse_object_prefix = %index.warehouse_object_prefix, - reason = %reason, - "deleted table warehouse index" - ); - Ok(()) - } - - async fn replace_stale_table_warehouse_index( - &self, - object: &str, - stale: &TableWarehouseIndexEntry, - replacement: &TableWarehouseIndexEntry, - reason: &'static str, - ) -> TableCatalogStoreResult { - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), object).await?; - let Some((current, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), object) - .await? - else { - return Ok(false); - }; - if current != *stale { - return Ok(false); - } - self.delete_warehouse_index_object_unlocked(object, stale, reason).await?; - self.write_entry_unlocked(self.catalog_bucket(), object, replacement, TableCatalogPutPrecondition::IfAbsent) - .await?; - Ok(true) - } - - async fn reserve_table_warehouse_index(&self, entry: &TableEntry) -> TableCatalogStoreResult { - let index = table_warehouse_index_entry(entry)?; - let object = self - .paths - .warehouse_index_entry_path(&index.table_bucket, &index.warehouse_object_prefix); - loop { - match self - .write_entry(self.catalog_bucket(), &object, &index, TableCatalogPutPrecondition::IfAbsent) - .await - { - Ok(()) => return Ok(WarehouseIndexReservation::Created), - Err(TableCatalogStoreError::Conflict(_)) => { - let Some((existing, _)) = self - .read_entry::(self.catalog_bucket(), &object) - .await? - else { - continue; - }; - if existing == index { - return Ok(WarehouseIndexReservation::AlreadyReserved); - } - if existing.table_bucket != index.table_bucket - || existing.warehouse_object_prefix != index.warehouse_object_prefix - || self.warehouse_index_entry_has_active_owner(&existing).await? - { - return Err(TableCatalogStoreError::Conflict(format!( - "table warehouse location is already registered: {}", - index.warehouse_object_prefix - ))); - } - if self - .replace_stale_table_warehouse_index(&object, &existing, &index, "stale reservation conflict") - .await? - { - return Ok(WarehouseIndexReservation::Created); - } - } - Err(err) => return Err(err), - } - } - } - - async fn delete_stale_table_warehouse_index( - &self, - object: &str, - index: &TableWarehouseIndexEntry, - reason: &'static str, - ) -> TableCatalogStoreResult<()> { - self.delete_warehouse_index_object(object, index, reason) - .await - .map(|_| ()) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to delete stale warehouse index {object}: {err}"))) - } - - async fn fail_closed_for_broken_warehouse_index( - &self, - object: &str, - index: &TableWarehouseIndexEntry, - reason: &'static str, - ) -> TableCatalogStoreResult> { - Err(TableCatalogStoreError::Internal(format!( - "active warehouse index {object} for {}/{}/{} ({}) is inconsistent: {reason}", - index.table_bucket, index.namespace, index.table, index.table_id - ))) - } - - async fn resolve_table_data_plane_resource_from_index_entry( - &self, - index_object: &str, - index: TableWarehouseIndexEntry, - ) -> TableCatalogStoreResult> { - if index.state != TableCatalogEntryState::Active { - return Ok(None); - } - let Some(table) = self.load_table(&index.table_bucket, &index.namespace, &index.table).await? else { - return self - .fail_closed_for_broken_warehouse_index(index_object, &index, "referenced table entry is missing") - .await; - }; - if table.state != TableCatalogEntryState::Active { - self.delete_stale_table_warehouse_index(index_object, &index, "referenced table is inactive") - .await?; - return Ok(None); - } - let current_prefix = table_warehouse_object_prefix(&table).map_err(|err| { - TableCatalogStoreError::Invalid(format!("warehouse index table entry has invalid location {index_object}: {err}")) - })?; - if current_prefix != index.warehouse_object_prefix { - self.delete_stale_table_warehouse_index(index_object, &index, "referenced table moved warehouse prefix") - .await?; - return Ok(None); - } - if table.table_id != index.table_id { - return self - .fail_closed_for_broken_warehouse_index(index_object, &index, "referenced table identity changed") - .await; - } - Ok(Some(table_data_plane_resource_from_entry(table, current_prefix))) - } - - async fn read_warehouse_index_state_unlocked(&self, table_bucket: &str) -> TableCatalogStoreResult { - let Some((state, _)) = self - .read_entry_unlocked::( - self.catalog_bucket(), - &self.paths.warehouse_index_state_path(table_bucket), - ) - .await? - else { - return Ok(false); - }; - Ok(state.version == TABLE_CATALOG_ENTRY_VERSION - && state.table_bucket == table_bucket - && state.state == TableCatalogEntryState::Active) - } - - async fn delete_created_table_warehouse_index( - &self, - entry: &TableEntry, - reservation: WarehouseIndexReservation, - reason: &'static str, - ) { - if reservation != WarehouseIndexReservation::Created { - return; - } - let warehouse_object_prefix = table_warehouse_object_prefix(entry).ok(); - if let Err(err) = self.delete_table_warehouse_index(entry).await { - tracing::warn!( - table_bucket = %entry.table_bucket, - namespace = %entry.namespace, - table = %entry.table, - table_id = %entry.table_id, - warehouse_object_prefix = warehouse_object_prefix.as_deref().unwrap_or(""), - reason = %reason, - error = %err, - "failed to roll back table warehouse index reservation" - ); - } - } - - async fn delete_table_warehouse_index(&self, entry: &TableEntry) -> TableCatalogStoreResult<()> { - let Ok(index) = table_warehouse_index_entry(entry) else { - return Ok(()); - }; - self.delete_warehouse_index_object( - &self - .paths - .warehouse_index_entry_path(&index.table_bucket, &index.warehouse_object_prefix), - &index, - "table warehouse index owner removed", - ) - .await - .map(|_| ()) - } - - async fn delete_table_warehouse_index_if_changed(&self, current: &TableEntry, next: &TableEntry) { - let Ok(current_index) = table_warehouse_index_entry(current) else { - return; - }; - let Ok(next_index) = table_warehouse_index_entry(next) else { - return; - }; - if current_index.warehouse_object_prefix == next_index.warehouse_object_prefix { - return; - } - if let Err(err) = self.delete_table_warehouse_index(current).await { - tracing::warn!( - table_bucket = %current.table_bucket, - namespace = %current.namespace, - table = %current.table, - table_id = %current.table_id, - warehouse_object_prefix = %current_index.warehouse_object_prefix, - error = %err, - "failed to delete stale table warehouse index" - ); - } - } - - async fn resolve_table_data_plane_resource_from_index( - &self, - table_bucket: &str, - object: &str, - ) -> TableCatalogStoreResult> { - for warehouse_object_prefix in warehouse_index_candidate_prefixes(object) { - let index_object = self.paths.warehouse_index_entry_path(table_bucket, warehouse_object_prefix); - let Some((index, _)) = self - .read_entry::(self.catalog_bucket(), &index_object) - .await? - else { - continue; - }; - if index.table_bucket != table_bucket || index.warehouse_object_prefix != warehouse_object_prefix { - return Err(TableCatalogStoreError::Invalid(format!( - "warehouse index entry does not match indexed prefix: {index_object}" - ))); - } - if let Some(resource) = self - .resolve_table_data_plane_resource_from_index_entry(&index_object, index) - .await? - { - return Ok(Some(resource)); - } - } - Ok(None) - } - - async fn backfill_active_table_warehouse_index( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult<()> { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((current, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Ok(()); - }; - if current.state != TableCatalogEntryState::Active { - return Ok(()); - } - self.reserve_table_warehouse_index(¤t).await.map(|_| ()) - } - - async fn backfill_table_warehouse_index(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { - let state_object = self.paths.warehouse_index_state_path(table_bucket); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &state_object).await?; - if self.read_warehouse_index_state_unlocked(table_bucket).await? { - return Ok(()); - } - for namespace in self.list_namespaces(table_bucket).await? { - if namespace.state != TableCatalogEntryState::Active { - continue; - } - for table in self.list_tables(table_bucket, &namespace.namespace).await? { - if table.state != TableCatalogEntryState::Active { - continue; - } - if let Err(err) = self - .backfill_active_table_warehouse_index(&table.table_bucket, &table.namespace, &table.table) - .await - { - if matches!(&err, TableCatalogStoreError::Invalid(_)) { - tracing::warn!( - table_bucket = %table.table_bucket, - namespace = %table.namespace, - table = %table.table, - table_id = %table.table_id, - error = %err, - "skipping invalid table warehouse location while backfilling warehouse index" - ); - continue; - } - return Err(err); - } - } - } - self.write_warehouse_index_state_unlocked(table_bucket).await - } - - async fn require_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { - if self.get_table_bucket(table_bucket).await?.is_none() { - return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); - } - Ok(()) - } - - async fn read_table_with_etag( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - ) -> TableCatalogStoreResult> { - let table_path = self.paths.table_entry_path(table_bucket, namespace, table); - let Some((entry, etag)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Ok(None); - }; - let Some(etag) = etag else { - return Err(TableCatalogStoreError::Internal(format!("catalog table entry has no etag: {table_path}"))); - }; - Ok(Some((entry, etag))) - } - - async fn read_table_with_etag_unlocked( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - ) -> TableCatalogStoreResult> { - let table_path = self.paths.table_entry_path(table_bucket, namespace, table); - let Some((entry, etag)) = self - .read_entry_unlocked::(self.catalog_bucket(), &table_path) - .await? - else { - return Ok(None); - }; - let Some(etag) = etag else { - return Err(TableCatalogStoreError::Internal(format!("catalog table entry has no etag: {table_path}"))); - }; - Ok(Some((entry, etag))) - } - - async fn read_view_with_etag_unlocked( - &self, - table_bucket: &str, - namespace: &Namespace, - view: &IdentifierSegment, - ) -> TableCatalogStoreResult> { - let view_path = self.paths.view_entry_path(table_bucket, namespace, view); - let Some((entry, etag)) = self - .read_entry_unlocked::(self.catalog_bucket(), &view_path) - .await? - else { - return Ok(None); - }; - let Some(etag) = etag else { - return Err(TableCatalogStoreError::Internal(format!("catalog view entry has no etag: {view_path}"))); - }; - Ok(Some((entry, etag))) - } - - async fn write_table_entry( - &self, - entry: TableEntry, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> { - validate_catalog_entry_version("table", entry.version)?; - self.require_table_bucket(&entry.table_bucket).await?; - let namespace = parse_namespace_for_store(&entry.namespace)?; - let table = parse_table_for_store(&entry.table)?; - validate_table_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; - let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); - let _namespace_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &namespace_path) - .await?; - if self - .read_entry_unlocked::(self.catalog_bucket(), &namespace_path) - .await? - .is_none() - { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - let table_path = self.paths.table_entry_path(&entry.table_bucket, &namespace, &table); - let _table_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let reservation = self.reserve_table_warehouse_index(&entry).await?; - let result = self - .write_entry_unlocked(self.catalog_bucket(), &table_path, &entry, precondition) - .await; - if result.is_err() { - self.delete_created_table_warehouse_index(&entry, reservation, "table entry write failed") - .await; - } - result - } - - async fn write_view_entry(&self, entry: ViewEntry, precondition: TableCatalogPutPrecondition) -> TableCatalogStoreResult<()> { - validate_catalog_entry_version("view", entry.version)?; - self.require_table_bucket(&entry.table_bucket).await?; - let namespace = parse_namespace_for_store(&entry.namespace)?; - let view = parse_table_for_store(&entry.view)?; - validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; - let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); - let _namespace_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &namespace_path) - .await?; - if self - .read_entry_unlocked::(self.catalog_bucket(), &namespace_path) - .await? - .is_none() - { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - let view_path = self.paths.view_entry_path(&entry.table_bucket, &namespace, &view); - let _view_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &view_path).await?; - self.write_entry_unlocked(self.catalog_bucket(), &view_path, &entry, precondition) - .await - } - - pub(crate) async fn get_external_catalog_bridge( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult> { - self.require_table_bucket(table_bucket).await?; - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - if self.get_namespace(table_bucket, &namespace.public_name()).await?.is_none() { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - table_bucket, - namespace.public_name() - ))); - } - let bridge_path = self.paths.external_catalog_bridge_path(table_bucket, &namespace, &table); - self.read_entry::(self.catalog_bucket(), &bridge_path) - .await - .map(|entry| entry.map(|(bridge, _)| bridge)) - } - - pub(crate) async fn put_external_catalog_bridge( - &self, - entry: ExternalCatalogBridgeEntry, - ) -> TableCatalogStoreResult { - validate_catalog_entry_version("external catalog bridge", entry.version)?; - self.require_table_bucket(&entry.table_bucket).await?; - self.ensure_object_backed_writes_allowed(&entry.table_bucket).await?; - let namespace = parse_namespace_for_store(&entry.namespace)?; - let table = parse_table_for_store(&entry.table)?; - if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - let bridge_path = self - .paths - .external_catalog_bridge_path(&entry.table_bucket, &namespace, &table); - self.write_entry(self.catalog_bucket(), &bridge_path, &entry, TableCatalogPutPrecondition::Any) - .await?; - Ok(entry) - } - - async fn read_commit_by_path(&self, object: &str) -> TableCatalogStoreResult> { - self.read_entry::(self.catalog_bucket(), object) - .await - .map(|entry| entry.map(|(commit, _)| commit)) - } - - async fn finalize_commit_log( - &self, - commit_path: &str, - idempotency_path: Option<&str>, - commit_log: &CommitLogEntry, - ) -> TableCatalogStoreResult<()> { - self.write_entry(self.catalog_bucket(), commit_path, commit_log, TableCatalogPutPrecondition::Any) - .await?; - if let Some(idempotency_path) = idempotency_path { - self.write_entry(self.catalog_bucket(), idempotency_path, commit_log, TableCatalogPutPrecondition::Any) - .await?; - } - Ok(()) - } - - async fn table_commit_recovery_report_for_entry( - &self, - entry: &TableEntry, - finalized_count: usize, - ) -> TableCatalogStoreResult { - let commit_prefix = self.paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id); - let mut commits = Vec::new(); - for object in self.backend.list_objects(self.catalog_bucket(), &commit_prefix).await? { - if !object.ends_with(".json") { - continue; - } - let Some(commit_log) = self.read_commit_by_path(&object).await? else { - continue; - }; - let idempotency_commit = match commit_log.idempotency_key.as_deref() { - Some(idempotency_key) => { - let idempotency_path = - self.paths - .commit_idempotency_entry_path(&entry.table_bucket, &entry.table_id, idempotency_key); - self.read_commit_by_path(&idempotency_path).await? - } - None => None, - }; - commits.push(table_commit_recovery_entry(entry, &commit_log, idempotency_commit.as_ref())); - } - commits.sort_by(|left, right| left.commit_id.cmp(&right.commit_id)); - - let finalization_required_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::FinalizationRequired)) - .count(); - let idempotency_repair_required_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::IdempotencyIndexRepairRequired)) - .count(); - let staged_before_table_update_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::StagedBeforeTableUpdate)) - .count(); - let manual_review_count = commits - .iter() - .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::ManualReview)) - .count(); - - Ok(TableCommitRecoveryReport { - table_bucket: entry.table_bucket.clone(), - namespace: entry.namespace.clone(), - table: entry.table.clone(), - table_id: entry.table_id.clone(), - current_metadata_location: entry.metadata_location.clone(), - current_version_token: entry.version_token.clone(), - current_generation: entry.generation, - commits, - staged_before_table_update_count, - finalization_required_count, - idempotency_repair_required_count, - manual_review_count, - finalized_count, - }) - } - - pub(crate) async fn plan_table_commit_recovery( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - self.table_commit_recovery_report_for_entry(&entry, 0).await - } - - pub(crate) async fn recover_table_commits( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - - let commit_prefix = self.paths.commit_log_entries_prefix(table_bucket, &entry.table_id); - let mut finalized_count = 0; - for commit_path in self.backend.list_objects(self.catalog_bucket(), &commit_prefix).await? { - if !commit_path.ends_with(".json") { - continue; - } - let Some(commit_log) = self.read_commit_by_path(&commit_path).await? else { - continue; - }; - let idempotency_path = commit_log.idempotency_key.as_deref().map(|idempotency_key| { - self.paths - .commit_idempotency_entry_path(table_bucket, &entry.table_id, idempotency_key) - }); - let idempotency_commit = match idempotency_path.as_deref() { - Some(idempotency_path) => self.read_commit_by_path(idempotency_path).await?, - None => None, - }; - let recovery_entry = table_commit_recovery_entry(&entry, &commit_log, idempotency_commit.as_ref()); - if matches!( - recovery_entry.recovery_state, - TableCommitRecoveryState::FinalizationRequired | TableCommitRecoveryState::IdempotencyIndexRepairRequired - ) { - let mut committed = commit_log; - committed.status = CommitLogStatus::Committed; - self.finalize_commit_log(&commit_path, idempotency_path.as_deref(), &committed) - .await?; - finalized_count += 1; - } - } - - self.table_commit_recovery_report_for_entry(&entry, finalized_count).await - } - - pub(crate) async fn get_table_maintenance_config( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - - let config_path = self - .paths - .table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id); - let config = self - .read_entry::(self.catalog_bucket(), &config_path) - .await? - .map(|(config, _)| config) - .unwrap_or_default(); - validate_table_maintenance_config(&config)?; - Ok(config) - } - - pub(crate) async fn put_table_bucket_maintenance_config( - &self, - table_bucket: &str, - config: TableMaintenanceConfig, - ) -> TableCatalogStoreResult { - validate_table_maintenance_config(&config)?; - self.require_table_bucket(table_bucket).await?; - self.ensure_object_backed_writes_allowed(table_bucket).await?; - let config_path = self.paths.table_bucket_maintenance_config_path(table_bucket); - self.write_entry(self.catalog_bucket(), &config_path, &config, TableCatalogPutPrecondition::Any) - .await?; - Ok(config) - } - - pub(crate) async fn get_effective_table_maintenance_config( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - - self.get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) - .await - } - - async fn get_effective_table_maintenance_config_for_entry_unlocked( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - entry: &TableEntry, - ) -> TableCatalogStoreResult { - let table_config_path = self - .paths - .table_maintenance_config_path(table_bucket, namespace, table, &entry.table_id); - if let Some((config, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), &table_config_path) - .await? - { - validate_table_maintenance_config(&config)?; - return Ok(TableMaintenanceEffectiveConfig { - config, - source: TableMaintenanceConfigSource::TableOverride, - }); - } - - let bucket_config_path = self.paths.table_bucket_maintenance_config_path(table_bucket); - if let Some((config, _)) = self - .read_entry_unlocked::(self.catalog_bucket(), &bucket_config_path) - .await? - { - validate_table_maintenance_config(&config)?; - return Ok(TableMaintenanceEffectiveConfig { - config, - source: TableMaintenanceConfigSource::TableBucketDefault, - }); - } - - Ok(TableMaintenanceEffectiveConfig { - config: TableMaintenanceConfig::default(), - source: TableMaintenanceConfigSource::Default, - }) - } - - pub(crate) async fn put_table_maintenance_config( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - config: TableMaintenanceConfig, - ) -> TableCatalogStoreResult { - validate_table_maintenance_config(&config)?; - self.ensure_object_backed_writes_allowed(table_bucket).await?; - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - - let config_path = self - .paths - .table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id); - self.write_entry(self.catalog_bucket(), &config_path, &config, TableCatalogPutPrecondition::Any) - .await?; - Ok(config) - } - - pub(crate) async fn put_table_metadata_maintenance_report( - &self, - report: &TableMetadataMaintenanceReport, - ) -> TableCatalogStoreResult<()> { - let report = table_maintenance_report_with_recommended_actions(report.clone()); - self.ensure_object_backed_writes_allowed(&report.job.table_bucket).await?; - let namespace = parse_namespace_for_store(&report.job.namespace)?; - let table = parse_table_for_store(&report.job.table)?; - let job_path = self.paths.table_maintenance_job_path( - &report.job.table_bucket, - &namespace, - &table, - &report.job.table_id, - &report.job.job_id, - ); - let latest_job_path = - self.paths - .table_maintenance_latest_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id); - let current_job_path = - self.paths - .table_maintenance_current_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id); - self.write_entry(self.catalog_bucket(), &job_path, &report, TableCatalogPutPrecondition::Any) - .await?; - self.write_entry(self.catalog_bucket(), &latest_job_path, &report, TableCatalogPutPrecondition::Any) - .await?; - self.write_entry(self.catalog_bucket(), ¤t_job_path, &report, TableCatalogPutPrecondition::Any) - .await - } - - pub(crate) async fn get_table_metadata_maintenance_report( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - job_id: &str, - ) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - - self.get_table_metadata_maintenance_report_for_entry_unlocked(table_bucket, &namespace, &table, &entry.table_id, job_id) - .await - } - - async fn get_table_metadata_maintenance_report_for_entry_unlocked( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - table_id: &str, - job_id: &str, - ) -> TableCatalogStoreResult> { - let job_path = self.table_metadata_maintenance_report_path(table_bucket, namespace, table, table_id, job_id); - self.read_entry_unlocked::(self.catalog_bucket(), &job_path) - .await - .map(|entry| entry.map(|(report, _)| table_maintenance_report_with_recommended_actions(report))) - } - - fn table_metadata_maintenance_report_path( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - table_id: &str, - job_id: &str, - ) -> String { - match job_id { - MAINTENANCE_JOB_ALIAS_LATEST => { - self.paths - .table_maintenance_latest_job_path(table_bucket, namespace, table, table_id) - } - MAINTENANCE_JOB_ALIAS_CURRENT => { - self.paths - .table_maintenance_current_job_path(table_bucket, namespace, table, table_id) - } - _ => self - .paths - .table_maintenance_job_path(table_bucket, namespace, table, table_id, job_id), - } - } - - pub(crate) async fn get_table_maintenance_scheduler_report( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - self.get_table_maintenance_scheduler_report_at(table_bucket, namespace, table, OffsetDateTime::now_utc()) - .await - } - - async fn get_table_maintenance_scheduler_report_at( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - now: OffsetDateTime, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - let effective = self - .get_effective_table_maintenance_config(table_bucket, &namespace.public_name(), table.as_str()) - .await?; - let current = self - .get_table_metadata_maintenance_report( - table_bucket, - &namespace.public_name(), - table.as_str(), - MAINTENANCE_JOB_ALIAS_CURRENT, - ) - .await?; - let reports = self - .list_table_metadata_maintenance_audit_reports(table_bucket, &namespace, &table, &entry.table_id) - .await?; - let quarantine = table_maintenance_scheduler_quarantine_boundary(&effective.config, &reports); - let mut recommended_actions = Vec::new(); - - let status = if !effective.config.background_enabled { - push_unique_maintenance_action( - &mut recommended_actions, - TableMaintenanceRecommendedAction::EnableBackgroundMaintenance, - ); - TableMaintenanceSchedulerStatus::Disabled - } else if effective.config.worker_paused { - push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::ResumeMaintenanceWorker); - TableMaintenanceSchedulerStatus::Paused - } else if let Some(current) = current.as_ref() - && matches!(current.job.status, TableMetadataMaintenanceJobStatus::Running) - && table_maintenance_job_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) - { - push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::WaitForActiveWorker); - TableMaintenanceSchedulerStatus::Backpressured - } else if let Some(current) = current.as_ref() - && matches!(current.job.status, TableMetadataMaintenanceJobStatus::Queued) - && table_maintenance_scheduler_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) - { - push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::RunMaintenanceWorker); - TableMaintenanceSchedulerStatus::Queued - } else if let Some(current) = current.as_ref() - && table_maintenance_job_retry_is_pending(¤t.job, now) - { - push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::WaitForRetryBackoff); - TableMaintenanceSchedulerStatus::RetryDeferred - } else if quarantine.active { - push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::ReviewQuarantine); - push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::InvestigateFailure); - TableMaintenanceSchedulerStatus::Quarantined - } else { - push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::NoActionRequired); - TableMaintenanceSchedulerStatus::Ready - }; - - Ok(TableMaintenanceSchedulerReport { - table_bucket: table_bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: entry.table_id, - status, - config_source: effective.source, - background_enabled: effective.config.background_enabled, - worker_paused: effective.config.worker_paused, - delete_enabled: effective.config.delete_enabled, - worker_lease_timeout_seconds: effective.config.worker_lease_timeout_seconds, - max_retry_attempts: effective.config.max_retry_attempts, - retry_initial_backoff_seconds: effective.config.retry_initial_backoff_seconds, - retry_max_backoff_seconds: effective.config.retry_max_backoff_seconds, - recommended_actions, - current_job: current.as_ref().map(table_maintenance_scheduler_job_summary), - quarantine, - audit_timeline: reports.iter().map(table_maintenance_scheduler_job_summary).collect(), - }) - } - - pub(crate) async fn run_table_maintenance_scheduler_once( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - scheduler_id: String, - ) -> TableCatalogStoreResult { - self.run_table_maintenance_scheduler_once_at(table_bucket, namespace, table, scheduler_id, OffsetDateTime::now_utc()) - .await - } - - async fn run_table_maintenance_scheduler_once_at( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - scheduler_id: String, - now: OffsetDateTime, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let namespace_name = namespace.public_name(); - let table_name = table.as_str().to_string(); - - let preflight = { - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, namespace_name, table_name - ))); - }; - let effective = self - .get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) - .await?; - self.table_metadata_maintenance_scheduler_preflight( - TableMaintenancePreflightContext { - table_bucket, - namespace: &namespace, - table: &table, - entry: &entry, - }, - &scheduler_id, - now, - effective, - ) - .await? - }; - let effective = match preflight { - TableMaintenanceSchedulerPreflight::Ready(effective) => effective, - TableMaintenanceSchedulerPreflight::Complete(report) => { - let scheduler = self - .get_table_maintenance_scheduler_report_at(table_bucket, &namespace_name, &table_name, now) - .await?; - return Ok(TableMaintenanceSchedulerRunResult { - report: *report, - scheduler, - }); - } - }; - - let mut report = self - .plan_table_metadata_maintenance( - table_bucket, - &namespace_name, - &table_name, - effective.config.retain_recent_metadata_files, - ) - .await?; - - let report = { - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, namespace_name, table_name - ))); - }; - let effective = self - .get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) - .await?; - match self - .table_metadata_maintenance_scheduler_preflight( - TableMaintenancePreflightContext { - table_bucket, - namespace: &namespace, - table: &table, - entry: &entry, - }, - &scheduler_id, - now, - effective, - ) - .await? - { - TableMaintenanceSchedulerPreflight::Ready(effective) => { - if report.job.retain_recent_metadata_files != effective.config.retain_recent_metadata_files { - return Err(TableCatalogStoreError::Conflict( - "maintenance config changed before scheduler claim".to_string(), - )); - } - if entry.metadata_location != report.current_metadata_location { - return Err(TableCatalogStoreError::Conflict( - "current metadata location changed before maintenance scheduler claim".to_string(), - )); - } - - let before_status = Some(report.job.status.clone()); - let before_quarantined_object_count = Some(report.job.quarantined_object_count); - let scheduled_at = maintenance_timestamp(now); - report.job.operation = if effective.config.delete_enabled { - TableMetadataMaintenanceOperation::Delete - } else { - TableMetadataMaintenanceOperation::DryRun - }; - report.job.status = TableMetadataMaintenanceJobStatus::Queued; - report.job.failure_reason = None; - report.job.config_source = effective.source; - report.job.scheduler_id = Some(scheduler_id); - report.job.scheduler_lease_id = Uuid::new_v4().to_string(); - report.job.scheduled_at = Some(scheduled_at); - report.job.worker_id = None; - report.job.lease_id = String::new(); - report.job.attempt = 0; - report.job.max_retry_attempts = effective.config.max_retry_attempts; - report.job.next_retry_after = None; - report.job.quarantine_enabled = effective.config.quarantine_enabled; - report.job.quarantine_retention_seconds = effective.config.quarantine_retention_seconds; - report.job.heartbeat_at = None; - report.job.started_at = None; - report.job.finished_at = None; - refresh_table_maintenance_report_recommended_actions(&mut report); - push_table_maintenance_audit_event( - &mut report, - now, - TableMaintenanceAuditActor::Scheduler, - TableMaintenanceAuditAction::SchedulerQueued, - None, - before_status, - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&report).await?; - report - } - TableMaintenanceSchedulerPreflight::Complete(report) => *report, - } - }; - - let scheduler = self - .get_table_maintenance_scheduler_report_at(table_bucket, &namespace_name, &table_name, now) - .await?; - Ok(TableMaintenanceSchedulerRunResult { report, scheduler }) - } - - pub(crate) async fn apply_table_maintenance_quarantine_operation( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - job_id: &str, - request: TableMaintenanceQuarantineOperationRequest, - ) -> TableCatalogStoreResult { - let action = request.action.clone(); - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let namespace_name = namespace.public_name(); - let table_name = table.as_str().to_string(); - - let report = if matches!(action, TableMaintenanceQuarantineAction::Inspect) { - self.get_table_metadata_maintenance_report(table_bucket, &namespace_name, &table_name, job_id) - .await? - .ok_or_else(|| { - TableCatalogStoreError::NotFound(format!( - "maintenance job {}/{}/{}/{}", - table_bucket, namespace_name, table_name, job_id - )) - })? - } else { - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, namespace_name, table_name - ))); - }; - let Some(mut report) = self - .get_table_metadata_maintenance_report_for_entry_unlocked( - table_bucket, - &namespace, - &table, - &entry.table_id, - MAINTENANCE_JOB_ALIAS_CURRENT, - ) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!( - "maintenance job {}/{}/{}/{}", - table_bucket, namespace_name, table_name, job_id - ))); - }; - if report.job.job_id != job_id { - return Err(TableCatalogStoreError::Conflict("maintenance job is not current".to_string())); - } - if !matches!(report.job.status, TableMetadataMaintenanceJobStatus::Failed) { - return Err(TableCatalogStoreError::Conflict( - "maintenance quarantine operation requires a failed job".to_string(), - )); - } - if !report.job.quarantine_enabled || report.job.quarantined_object_count == 0 { - return Err(TableCatalogStoreError::Conflict( - "maintenance job has no active quarantine boundary".to_string(), - )); - } - - let before_status = Some(report.job.status.clone()); - let before_quarantined_object_count = Some(report.job.quarantined_object_count); - report.job.quarantined_object_count = 0; - match &action { - TableMaintenanceQuarantineAction::Inspect => unreachable!("inspect branch handled before mutation"), - TableMaintenanceQuarantineAction::Release => { - report.job.failure_reason = - Some(table_maintenance_quarantine_operator_reason("released", request.reason.as_deref())); - } - TableMaintenanceQuarantineAction::Retry => { - report.job.next_retry_after = None; - report.job.failure_reason = Some(table_maintenance_quarantine_operator_reason( - "released for retry", - request.reason.as_deref(), - )); - } - TableMaintenanceQuarantineAction::Abandon => { - report.job.next_retry_after = None; - report.job.failure_reason = - Some(table_maintenance_quarantine_operator_reason("abandoned", request.reason.as_deref())); - } - } - refresh_table_maintenance_report_recommended_actions(&mut report); - let audit_action = match &action { - TableMaintenanceQuarantineAction::Inspect => unreachable!("inspect branch handled before mutation"), - TableMaintenanceQuarantineAction::Release => TableMaintenanceAuditAction::QuarantineRelease, - TableMaintenanceQuarantineAction::Retry => TableMaintenanceAuditAction::QuarantineRetry, - TableMaintenanceQuarantineAction::Abandon => TableMaintenanceAuditAction::QuarantineAbandon, - }; - push_table_maintenance_audit_event( - &mut report, - OffsetDateTime::now_utc(), - TableMaintenanceAuditActor::Operator, - audit_action, - request.reason, - before_status, - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&report).await?; - report - }; - - let scheduler = self - .get_table_maintenance_scheduler_report(table_bucket, &namespace_name, &table_name) - .await?; - Ok(TableMaintenanceQuarantineOperationResult { - action, - report, - scheduler, - }) - } - - async fn list_table_metadata_maintenance_audit_reports( - &self, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - table_id: &str, - ) -> TableCatalogStoreResult> { - let jobs_prefix = self - .paths - .table_maintenance_jobs_prefix(table_bucket, namespace, table, table_id); - let mut reports = Vec::new(); - for object in self.backend.list_objects(self.catalog_bucket(), &jobs_prefix).await? { - if !object.ends_with(".json") { - continue; - } - if let Some((report, _)) = self - .read_entry::(self.catalog_bucket(), &object) - .await? - { - reports.push(table_maintenance_report_with_recommended_actions(report)); - } - } - reports.sort_by(|left, right| { - table_maintenance_report_order_timestamp(right) - .cmp(&table_maintenance_report_order_timestamp(left)) - .then_with(|| left.job.job_id.cmp(&right.job.job_id)) - }); - reports.truncate(TABLE_MAINTENANCE_SCHEDULER_AUDIT_LIMIT); - Ok(reports) - } - - async fn table_metadata_maintenance_scheduler_preflight( - &self, - context: TableMaintenancePreflightContext<'_>, - scheduler_id: &str, - now: OffsetDateTime, - effective: TableMaintenanceEffectiveConfig, - ) -> TableCatalogStoreResult { - if !effective.config.background_enabled { - let report = self - .put_table_metadata_maintenance_scheduler_control_report(TableMaintenanceSchedulerControlReport { - table_bucket: context.table_bucket, - namespace: context.namespace, - table: context.table, - entry: context.entry, - scheduler_id: scheduler_id.to_string(), - effective: &effective, - status: TableMetadataMaintenanceJobStatus::Disabled, - reason: "background maintenance is disabled", - now, - }) - .await?; - return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(report))); - } - if effective.config.worker_paused { - let report = self - .put_table_metadata_maintenance_scheduler_control_report(TableMaintenanceSchedulerControlReport { - table_bucket: context.table_bucket, - namespace: context.namespace, - table: context.table, - entry: context.entry, - scheduler_id: scheduler_id.to_string(), - effective: &effective, - status: TableMetadataMaintenanceJobStatus::Paused, - reason: "background maintenance worker is paused", - now, - }) - .await?; - return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(report))); - } - - if let Some(current) = self - .get_table_metadata_maintenance_report_for_entry_unlocked( - context.table_bucket, - context.namespace, - context.table, - &context.entry.table_id, - MAINTENANCE_JOB_ALIAS_CURRENT, - ) - .await? - { - if matches!(current.job.status, TableMetadataMaintenanceJobStatus::Running) { - if table_maintenance_job_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) { - return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(current))); - } - self.expire_table_maintenance_job( - current, - now, - "maintenance worker lease expired", - TableMaintenanceAuditAction::WorkerLeaseExpired, - ) - .await?; - } else if matches!(current.job.status, TableMetadataMaintenanceJobStatus::Queued) { - if table_maintenance_scheduler_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) { - return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(current))); - } - self.expire_table_maintenance_job( - current, - now, - "maintenance scheduler lease expired", - TableMaintenanceAuditAction::SchedulerLeaseExpired, - ) - .await?; - } else if table_maintenance_job_retry_is_pending(¤t.job, now) { - return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(current))); - } - } - - Ok(TableMaintenanceSchedulerPreflight::Ready(effective)) - } - - pub(crate) async fn run_table_metadata_maintenance_worker_once( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - worker_id: String, - ) -> TableCatalogStoreResult { - self.run_table_metadata_maintenance_worker_once_at(table_bucket, namespace, table, worker_id, OffsetDateTime::now_utc()) - .await - } - - async fn run_table_metadata_maintenance_worker_once_at( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - worker_id: String, - now: OffsetDateTime, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let namespace_name = namespace.public_name(); - let table_name = table.as_str().to_string(); - - let (effective, queued) = { - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, namespace_name, table_name - ))); - }; - let effective = self - .get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) - .await?; - match self - .table_metadata_maintenance_worker_preflight( - TableMaintenancePreflightContext { - table_bucket, - namespace: &namespace, - table: &table, - entry: &entry, - }, - &worker_id, - now, - effective, - ) - .await? - { - TableMaintenanceWorkerPreflight::Ready { effective, queued } => (effective, queued), - TableMaintenanceWorkerPreflight::Complete(report) => return Ok(*report), - } - }; - - let mut report = if let Some(queued) = queued { - *queued - } else { - self.plan_table_metadata_maintenance( - table_bucket, - &namespace_name, - &table_name, - effective.config.retain_recent_metadata_files, - ) - .await? - }; - - let (report, effective, delete) = { - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, namespace_name, table_name - ))); - }; - let effective = self - .get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) - .await?; - let (effective, queued) = match self - .table_metadata_maintenance_worker_preflight( - TableMaintenancePreflightContext { - table_bucket, - namespace: &namespace, - table: &table, - entry: &entry, - }, - &worker_id, - now, - effective, - ) - .await? - { - TableMaintenanceWorkerPreflight::Ready { effective, queued } => (effective, queued), - TableMaintenanceWorkerPreflight::Complete(report) => return Ok(*report), - }; - if let Some(queued) = queued { - if queued.job.job_id != report.job.job_id { - return Err(TableCatalogStoreError::Conflict( - "queued maintenance job changed before worker claim".to_string(), - )); - } - report = *queued; - } - - if report.job.retain_recent_metadata_files != effective.config.retain_recent_metadata_files { - return Err(TableCatalogStoreError::Conflict( - "maintenance config changed before worker claim".to_string(), - )); - } - if entry.metadata_location != report.current_metadata_location { - return Err(TableCatalogStoreError::Conflict( - "current metadata location changed before maintenance worker claim".to_string(), - )); - } - - let was_queued_claim = matches!(report.job.status, TableMetadataMaintenanceJobStatus::Queued); - let before_status = Some(report.job.status.clone()); - let before_quarantined_object_count = Some(report.job.quarantined_object_count); - let started_at = maintenance_timestamp(now); - if !was_queued_claim { - report.job.operation = if effective.config.delete_enabled { - TableMetadataMaintenanceOperation::Delete - } else { - TableMetadataMaintenanceOperation::DryRun - }; - } - report.job.status = TableMetadataMaintenanceJobStatus::Running; - report.job.failure_reason = None; - report.job.config_source = effective.source; - report.job.worker_id = Some(worker_id); - report.job.lease_id = Uuid::new_v4().to_string(); - report.job.attempt = 1; - report.job.max_retry_attempts = effective.config.max_retry_attempts; - report.job.next_retry_after = None; - report.job.quarantine_enabled = effective.config.quarantine_enabled; - report.job.quarantine_retention_seconds = effective.config.quarantine_retention_seconds; - report.job.heartbeat_at = Some(started_at.clone()); - report.job.started_at = Some(started_at); - report.job.finished_at = None; - refresh_table_maintenance_report_recommended_actions(&mut report); - push_table_maintenance_audit_event( - &mut report, - now, - TableMaintenanceAuditActor::Worker, - TableMaintenanceAuditAction::WorkerStarted, - None, - before_status, - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&report).await?; - - let delete = matches!(report.job.operation, TableMetadataMaintenanceOperation::Delete); - (report, effective, delete) - }; - - self.finish_table_metadata_maintenance_run(table_bucket, &namespace_name, &table_name, delete, &effective, report) - .await - } - - async fn table_metadata_maintenance_worker_preflight( - &self, - context: TableMaintenancePreflightContext<'_>, - worker_id: &str, - now: OffsetDateTime, - effective: TableMaintenanceEffectiveConfig, - ) -> TableCatalogStoreResult { - if !effective.config.background_enabled { - let report = self - .put_table_metadata_maintenance_worker_control_report(TableMaintenanceWorkerControlReport { - table_bucket: context.table_bucket, - namespace: context.namespace, - table: context.table, - entry: context.entry, - worker_id: worker_id.to_string(), - effective: &effective, - status: TableMetadataMaintenanceJobStatus::Disabled, - reason: "background maintenance is disabled", - now, - }) - .await?; - return Ok(TableMaintenanceWorkerPreflight::Complete(Box::new(report))); - } - if effective.config.worker_paused { - let report = self - .put_table_metadata_maintenance_worker_control_report(TableMaintenanceWorkerControlReport { - table_bucket: context.table_bucket, - namespace: context.namespace, - table: context.table, - entry: context.entry, - worker_id: worker_id.to_string(), - effective: &effective, - status: TableMetadataMaintenanceJobStatus::Paused, - reason: "background maintenance worker is paused", - now, - }) - .await?; - return Ok(TableMaintenanceWorkerPreflight::Complete(Box::new(report))); - } - - if let Some(current) = self - .get_table_metadata_maintenance_report_for_entry_unlocked( - context.table_bucket, - context.namespace, - context.table, - &context.entry.table_id, - MAINTENANCE_JOB_ALIAS_CURRENT, - ) - .await? - { - if matches!(current.job.status, TableMetadataMaintenanceJobStatus::Running) { - if table_maintenance_job_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) { - return Ok(TableMaintenanceWorkerPreflight::Complete(Box::new(current))); - } - self.expire_table_maintenance_job( - current, - now, - "maintenance worker lease expired", - TableMaintenanceAuditAction::WorkerLeaseExpired, - ) - .await?; - } else if matches!(current.job.status, TableMetadataMaintenanceJobStatus::Queued) { - if table_maintenance_scheduler_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) { - return Ok(TableMaintenanceWorkerPreflight::Ready { - effective, - queued: Some(Box::new(current)), - }); - } - self.expire_table_maintenance_job( - current, - now, - "maintenance scheduler lease expired", - TableMaintenanceAuditAction::SchedulerLeaseExpired, - ) - .await?; - } else if table_maintenance_job_retry_is_pending(¤t.job, now) { - return Ok(TableMaintenanceWorkerPreflight::Complete(Box::new(current))); - } - } - - Ok(TableMaintenanceWorkerPreflight::Ready { effective, queued: None }) - } - - pub(crate) async fn heartbeat_table_metadata_maintenance_job( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - job_id: &str, - lease_id: &str, - worker_id: &str, - ) -> TableCatalogStoreResult { - self.heartbeat_table_metadata_maintenance_job_at( - TableMaintenanceHeartbeatRef { - table_bucket, - namespace, - table, - job_id, - lease_id, - worker_id, - }, - OffsetDateTime::now_utc(), - ) - .await - } - - async fn heartbeat_table_metadata_maintenance_job_at( - &self, - heartbeat: TableMaintenanceHeartbeatRef<'_>, - now: OffsetDateTime, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(heartbeat.namespace)?; - let table = parse_table_for_store(heartbeat.table)?; - let table_path = self.paths.table_entry_path(heartbeat.table_bucket, &namespace, &table); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((entry, _)) = self - .read_table_with_etag_unlocked(heartbeat.table_bucket, &namespace, &table) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - heartbeat.table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - let Some(mut report) = self - .get_table_metadata_maintenance_report_for_entry_unlocked( - heartbeat.table_bucket, - &namespace, - &table, - &entry.table_id, - MAINTENANCE_JOB_ALIAS_CURRENT, - ) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!( - "maintenance job {}/{}/{}/{}", - heartbeat.table_bucket, - namespace.public_name(), - table.as_str(), - heartbeat.job_id - ))); - }; - if report.job.job_id != heartbeat.job_id { - return Err(TableCatalogStoreError::Conflict("maintenance job is not current".to_string())); - } - if !matches!(report.job.status, TableMetadataMaintenanceJobStatus::Running) { - return Err(TableCatalogStoreError::Conflict("maintenance job is not running".to_string())); - } - if report.job.lease_id != heartbeat.lease_id { - return Err(TableCatalogStoreError::Conflict("maintenance lease does not match".to_string())); - } - if report.job.worker_id.as_deref() != Some(heartbeat.worker_id) { - return Err(TableCatalogStoreError::Conflict("maintenance worker does not match".to_string())); - } - - report.job.heartbeat_at = Some(maintenance_timestamp(now)); - refresh_table_maintenance_report_recommended_actions(&mut report); - let before_quarantined_object_count = Some(report.job.quarantined_object_count); - push_table_maintenance_audit_event( - &mut report, - now, - TableMaintenanceAuditActor::Worker, - TableMaintenanceAuditAction::WorkerHeartbeat, - None, - Some(TableMetadataMaintenanceJobStatus::Running), - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&report).await?; - Ok(report) - } - - async fn expire_table_maintenance_job( - &self, - mut report: TableMetadataMaintenanceReport, - now: OffsetDateTime, - reason: &str, - action: TableMaintenanceAuditAction, - ) -> TableCatalogStoreResult { - let before_status = Some(report.job.status.clone()); - let before_quarantined_object_count = Some(report.job.quarantined_object_count); - report.job.status = TableMetadataMaintenanceJobStatus::Failed; - report.job.failure_reason = Some(reason.to_string()); - report.job.finished_at = Some(maintenance_timestamp(now)); - refresh_table_maintenance_report_recommended_actions(&mut report); - push_table_maintenance_audit_event( - &mut report, - now, - TableMaintenanceAuditActor::Scheduler, - action, - Some(reason.to_string()), - before_status, - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&report).await?; - Ok(report) - } - - async fn put_table_metadata_maintenance_scheduler_control_report( - &self, - control: TableMaintenanceSchedulerControlReport<'_>, - ) -> TableCatalogStoreResult { - let timestamp = maintenance_timestamp(control.now); - let cleanup_watermark_unix_seconds = - (control.now - Duration::seconds(TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS)).unix_timestamp(); - let current_metadata_location = control.entry.metadata_location.clone(); - let report = TableMetadataMaintenanceReport { - job: TableMetadataMaintenanceJob { - job_id: Uuid::new_v4().to_string(), - table_bucket: control.table_bucket.to_string(), - namespace: control.namespace.public_name(), - table: control.table.as_str().to_string(), - table_id: control.entry.table_id.clone(), - operation: TableMetadataMaintenanceOperation::DryRun, - status: control.status, - failure_reason: Some(control.reason.to_string()), - recommended_actions: Vec::new(), - config_source: control.effective.source, - scheduler_id: Some(control.scheduler_id), - scheduler_lease_id: String::new(), - scheduled_at: Some(timestamp.clone()), - worker_id: None, - lease_id: String::new(), - attempt: 0, - max_retry_attempts: control.effective.config.max_retry_attempts, - next_retry_after: None, - quarantine_enabled: control.effective.config.quarantine_enabled, - quarantine_retention_seconds: control.effective.config.quarantine_retention_seconds, - heartbeat_at: None, - started_at: None, - finished_at: Some(timestamp), - current_metadata_location: current_metadata_location.clone(), - current_generation: control.entry.generation, - retain_recent_metadata_files: control.effective.config.retain_recent_metadata_files, - safety_window_seconds: TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS, - cleanup_watermark_unix_seconds, - planned_metadata_file_count: 0, - retained_metadata_file_count: 0, - cleanup_candidate_count: 0, - deletable_metadata_file_count: 0, - deleted_metadata_file_count: 0, - planned_object_file_count: 0, - cleanup_candidate_object_count: 0, - deletable_object_count: 0, - deleted_object_count: 0, - quarantined_object_count: 0, - }, - current_metadata_location, - retained_metadata_locations: Vec::new(), - cleanup_candidate_locations: Vec::new(), - deletable_metadata_locations: Vec::new(), - cleanup_object_candidate_locations: Vec::new(), - deletable_object_locations: Vec::new(), - object_reports: Vec::new(), - object_cleanup_reports: Vec::new(), - referenced_object_reports: Vec::new(), - reachability_graph: TableMaintenanceReachabilityGraphReport::default(), - snapshot_expiration: None, - compaction: None, - audit_events: Vec::new(), - }; - let mut report = table_maintenance_report_with_recommended_actions(report); - push_table_maintenance_audit_event( - &mut report, - control.now, - TableMaintenanceAuditActor::Scheduler, - TableMaintenanceAuditAction::SchedulerControl, - Some(control.reason.to_string()), - None, - None, - ); - self.put_table_metadata_maintenance_report(&report).await?; - Ok(report) - } - - async fn put_table_metadata_maintenance_worker_control_report( - &self, - control: TableMaintenanceWorkerControlReport<'_>, - ) -> TableCatalogStoreResult { - let timestamp = maintenance_timestamp(control.now); - let cleanup_watermark_unix_seconds = - (control.now - Duration::seconds(TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS)).unix_timestamp(); - let current_metadata_location = control.entry.metadata_location.clone(); - let report = TableMetadataMaintenanceReport { - job: TableMetadataMaintenanceJob { - job_id: Uuid::new_v4().to_string(), - table_bucket: control.table_bucket.to_string(), - namespace: control.namespace.public_name(), - table: control.table.as_str().to_string(), - table_id: control.entry.table_id.clone(), - operation: TableMetadataMaintenanceOperation::DryRun, - status: control.status, - failure_reason: Some(control.reason.to_string()), - recommended_actions: Vec::new(), - config_source: control.effective.source, - scheduler_id: None, - scheduler_lease_id: String::new(), - scheduled_at: None, - worker_id: Some(control.worker_id), - lease_id: String::new(), - attempt: 0, - max_retry_attempts: control.effective.config.max_retry_attempts, - next_retry_after: None, - quarantine_enabled: control.effective.config.quarantine_enabled, - quarantine_retention_seconds: control.effective.config.quarantine_retention_seconds, - heartbeat_at: None, - started_at: Some(timestamp.clone()), - finished_at: Some(timestamp), - current_metadata_location: current_metadata_location.clone(), - current_generation: control.entry.generation, - retain_recent_metadata_files: control.effective.config.retain_recent_metadata_files, - safety_window_seconds: TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS, - cleanup_watermark_unix_seconds, - planned_metadata_file_count: 0, - retained_metadata_file_count: 0, - cleanup_candidate_count: 0, - deletable_metadata_file_count: 0, - deleted_metadata_file_count: 0, - planned_object_file_count: 0, - cleanup_candidate_object_count: 0, - deletable_object_count: 0, - deleted_object_count: 0, - quarantined_object_count: 0, - }, - current_metadata_location, - retained_metadata_locations: Vec::new(), - cleanup_candidate_locations: Vec::new(), - deletable_metadata_locations: Vec::new(), - cleanup_object_candidate_locations: Vec::new(), - deletable_object_locations: Vec::new(), - object_reports: Vec::new(), - object_cleanup_reports: Vec::new(), - referenced_object_reports: Vec::new(), - reachability_graph: TableMaintenanceReachabilityGraphReport::default(), - snapshot_expiration: None, - compaction: None, - audit_events: Vec::new(), - }; - let mut report = table_maintenance_report_with_recommended_actions(report); - push_table_maintenance_audit_event( - &mut report, - control.now, - TableMaintenanceAuditActor::Scheduler, - TableMaintenanceAuditAction::WorkerControl, - Some(control.reason.to_string()), - None, - None, - ); - self.put_table_metadata_maintenance_report(&report).await?; - Ok(report) - } - - pub(crate) async fn plan_table_snapshot_expiration( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - config: TableSnapshotExpirationConfig, - ) -> TableCatalogStoreResult { - validate_table_snapshot_expiration_config(&config)?; - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { - return Err(TableCatalogStoreError::Invalid( - "current metadata location must be inside the table metadata directory".to_string(), - )); - } - - let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "current metadata object {}", - entry.metadata_location - ))); - }; - let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { - TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) - })?; - if !current_metadata.is_object() { - return Err(TableCatalogStoreError::Invalid(format!( - "current metadata {} must be a JSON object", - entry.metadata_location - ))); - } - - Ok(table_snapshot_expiration_report( - table_bucket, - &namespace, - &table, - &entry, - ¤t_metadata, - config, - OffsetDateTime::now_utc(), - )) - } - - pub(crate) async fn plan_table_compaction( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - config: TableCompactionPlanningConfig, - ) -> TableCatalogStoreResult { - validate_table_compaction_planning_config(&config)?; - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { - return Err(TableCatalogStoreError::Invalid( - "current metadata location must be inside the table metadata directory".to_string(), - )); - } - - let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "current metadata object {}", - entry.metadata_location - ))); - }; - let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { - TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) - })?; - if !current_metadata.is_object() { - return Err(TableCatalogStoreError::Invalid(format!( - "current metadata {} must be a JSON object", - entry.metadata_location - ))); - } - - table_compaction_planning_report(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata, config).await - } - - pub(crate) async fn commit_table_compaction( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - config: TableCompactionPlanningConfig, - ) -> TableCatalogStoreResult { - validate_table_compaction_planning_config(&config)?; - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { - return Err(TableCatalogStoreError::Invalid( - "current metadata location must be inside the table metadata directory".to_string(), - )); - } - - let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "current metadata object {}", - entry.metadata_location - ))); - }; - let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { - TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) - })?; - if !current_metadata.is_object() { - return Err(TableCatalogStoreError::Invalid(format!( - "current metadata {} must be a JSON object", - entry.metadata_location - ))); - } - let mut report = - table_compaction_planning_report(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata, config) - .await?; - if report.status != TableCompactionPlanningStatus::RewriteCandidates { - return Err(TableCatalogStoreError::Invalid("compaction has no safe rewrite candidates".to_string())); - } - let current_data_files = - compaction_current_data_files(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata).await?; - let current_data_files_by_key = current_data_files - .iter() - .map(|file| (file.object_key.as_str(), file)) - .collect::>(); - let rewritten_inputs = report - .rewrite_groups - .iter() - .flat_map(|group| group.input_file_locations.iter().cloned()) - .collect::>(); - let mut manifest_data_files = current_data_files - .iter() - .filter(|file| !rewritten_inputs.contains(&file.object_key)) - .cloned() - .collect::>(); - - let now = OffsetDateTime::now_utc(); - let snapshot_id = compaction_snapshot_id(¤t_metadata, &entry, now); - let sequence_number = next_compaction_sequence_number(¤t_metadata); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let warehouse_object_prefix = table_warehouse_object_prefix(&entry)?; - let compaction_id = Uuid::new_v4().to_string(); - let mut compacted_files = Vec::with_capacity(report.rewrite_groups.len()); - for rewrite_group in &mut report.rewrite_groups { - let output_prefix = rewrite_group - .input_file_locations - .first() - .and_then(|input| compaction_data_file_rewrite_prefix(&namespace, &table, Some(&warehouse_object_prefix), input)) - .ok_or_else(|| TableCatalogStoreError::Invalid("compaction rewrite group has no input files".to_string()))?; - let output_file = format!("{output_prefix}/compaction-{compaction_id}-{}.parquet", rewrite_group.group_id); - let output_file_path = table_object_s3_location(table_bucket, &output_file); - let (partition_spec_id, partition) = compaction_rewrite_group_partition(¤t_data_files_by_key, rewrite_group)?; - let sort_order_id = compaction_rewrite_group_sort_order(¤t_data_files_by_key, rewrite_group)?; - let mut input_files = Vec::with_capacity(rewrite_group.input_file_locations.len()); - for input_file in &rewrite_group.input_file_locations { - let Some(input_object) = self.backend.read_object(table_bucket, input_file).await? else { - return Err(TableCatalogStoreError::NotFound(format!("compaction input data file {input_file}"))); - }; - input_files.push((input_file.clone(), input_object.data)); - } - let compacted_file = compact_parquet_data_files(&input_files)?; - let output_bytes = u64::try_from(compacted_file.data.len()).unwrap_or(u64::MAX); - self.backend - .put_object(table_bucket, &output_file, compacted_file.data, TableCatalogPutPrecondition::IfAbsent) - .await?; - rewrite_group.output_file_location = Some(output_file_path.clone()); - rewrite_group.output_bytes = Some(output_bytes); - compacted_files.push(CompactedDataFile { - object_key: output_file, - file_path: output_file_path, - file_size_bytes: output_bytes, - record_count: compacted_file.record_count, - partition_spec_id, - partition, - sort_order_id, - status: 1, - snapshot_id, - sequence_number, - file_sequence_number: sequence_number, - }); - } - manifest_data_files.extend(compacted_files.iter().cloned()); - - let new_manifest = format!("{metadata_dir}/manifest-compaction-{compaction_id}.avro"); - let new_manifest_list = format!("{metadata_dir}/snap-{snapshot_id}-compaction-{compaction_id}.avro"); - let new_metadata = - default_table_metadata_file_path(&namespace, &table, &format!("compaction-{compaction_id}.metadata.json")); - let manifest_data = compacted_manifest_avro_bytes(&manifest_data_files)?; - let manifest_length = u64::try_from(manifest_data.len()).unwrap_or(u64::MAX); - self.backend - .put_object(table_bucket, &new_manifest, manifest_data, TableCatalogPutPrecondition::IfAbsent) - .await?; - let added_files_count = compacted_files.len(); - let added_rows_count = compacted_files - .iter() - .fold(0_u64, |rows, file| rows.saturating_add(file.record_count)); - let existing_files_count = manifest_data_files.len().saturating_sub(added_files_count); - let existing_rows_count = manifest_data_files - .iter() - .filter(|file| file.status == 0) - .fold(0_u64, |rows, file| rows.saturating_add(file.record_count)); - let manifest_list_data = compacted_manifest_list_avro_bytes(CompactionManifestListSummary { - manifest_path: &new_manifest, - manifest_length, - partition_spec_id: compaction_manifest_partition_spec_id(&manifest_data_files)?, - snapshot_id, - sequence_number, - added_files_count, - existing_files_count, - added_rows_count, - existing_rows_count, - })?; - self.backend - .put_object( - table_bucket, - &new_manifest_list, - manifest_list_data, - TableCatalogPutPrecondition::IfAbsent, - ) - .await?; - let new_metadata_data = compaction_metadata_json( - ¤t_metadata, - &entry, - snapshot_id, - sequence_number, - &new_manifest_list, - &entry.metadata_location, - now, - )?; - self.backend - .put_object(table_bucket, &new_metadata, new_metadata_data, TableCatalogPutPrecondition::IfAbsent) - .await?; - - let commit_result = self - .commit_table(TableCommitRequest { - table_bucket: table_bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: format!("compaction-{compaction_id}"), - idempotency_key: Some(format!("compaction-{compaction_id}")), - operation: "compaction".to_string(), - expected_version_token: entry.version_token, - expected_metadata_location: entry.metadata_location, - new_metadata_location: new_metadata.clone(), - requirements: Vec::new(), - writer: Some("rustfs-maintenance".to_string()), - }) - .await?; - - report.status = TableCompactionPlanningStatus::Committed; - report.committed_metadata_location = Some(commit_result.table.metadata_location); - for snapshot in &mut report.snapshot_reports { - if snapshot.status == TableCompactionPlanningStatus::RewriteCandidates { - snapshot.status = TableCompactionPlanningStatus::Committed; - if !snapshot.reasons.contains(&TableCompactionPlanningReason::CompactionCommitted) { - snapshot.reasons.push(TableCompactionPlanningReason::CompactionCommitted); - } - } - } - Ok(report) - } - - pub(crate) async fn export_table_catalog_entry( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - - let Some((table_bucket_entry, _)) = self - .read_entry::(self.catalog_bucket(), &self.paths.table_bucket_entry_path(table_bucket)) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); - }; - let Some((namespace_entry, _)) = self - .read_entry::(self.catalog_bucket(), &self.paths.namespace_entry_path(table_bucket, &namespace)) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - table_bucket, - namespace.public_name() - ))); - }; - let Some((table_entry, _)) = self - .read_entry::(self.catalog_bucket(), &self.paths.table_entry_path(table_bucket, &namespace, &table)) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - let commit_recovery = self.table_commit_recovery_report_for_entry(&table_entry, 0).await?; - let backing_manifest = table_catalog_backing_manifest(&self.paths, &namespace, &table, &table_entry, &commit_recovery); - - Ok(TableCatalogExport { - table_bucket: table_bucket_entry, - namespace: namespace_entry, - table: table_entry, - backing_manifest, - }) - } - - pub(crate) async fn plan_durable_strong_backing_migration( - &self, - table_bucket: &str, - ) -> TableCatalogStoreResult { - if self.get_table_bucket(table_bucket).await?.is_none() { - return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); - } - - let namespaces = self.list_namespaces(table_bucket).await?; - let mut table_count: usize = 0; - let mut view_count: usize = 0; - let mut commit_log_count: usize = 0; - let mut idempotency_index_count: usize = 0; - let mut recovery_required_count: usize = 0; - let mut manual_review_count: usize = 0; - let mut warehouse_prefix_owners = BTreeMap::::new(); - - for namespace in &namespaces { - let tables = self.list_tables(table_bucket, &namespace.namespace).await?; - for table in tables { - table_count += 1; - if table.state == TableCatalogEntryState::Active { - let warehouse_prefix = table_warehouse_object_prefix(&table)?; - warehouse_prefix_owners - .entry(warehouse_prefix) - .and_modify(|count| *count = count.saturating_add(1)) - .or_insert(1); - } - - let recovery = self.table_commit_recovery_report_for_entry(&table, 0).await?; - commit_log_count = commit_log_count.saturating_add(recovery.commits.len()); - idempotency_index_count = idempotency_index_count.saturating_add( - self.backend - .list_objects( - self.catalog_bucket(), - &self.paths.commit_idempotency_entries_prefix(table_bucket, &table.table_id), - ) - .await? - .into_iter() - .filter(|object| object.ends_with(".json")) - .count(), - ); - recovery_required_count = recovery_required_count - .saturating_add(recovery.staged_before_table_update_count) - .saturating_add(recovery.finalization_required_count) - .saturating_add(recovery.idempotency_repair_required_count); - manual_review_count = manual_review_count.saturating_add(recovery.manual_review_count); - } - view_count = view_count.saturating_add(self.list_views(table_bucket, &namespace.namespace).await?.len()); - } - - let warehouse_index_ready = self.warehouse_index_ready(table_bucket).await?; - let duplicate_warehouse_prefix_count = warehouse_prefix_owners.values().filter(|count| **count > 1).count(); - let mut blockers = Vec::new(); - let mut recommended_actions = Vec::new(); - if recovery_required_count > 0 { - blockers.push(TableCatalogBackingMigrationBlocker::CommitRecoveryRequired); - } - if manual_review_count > 0 { - blockers.push(TableCatalogBackingMigrationBlocker::CommitManualReviewRequired); - } - if recovery_required_count > 0 || manual_review_count > 0 { - recommended_actions.push(TableCatalogBackingMigrationAction::RunCatalogRecovery); - } - if !warehouse_index_ready { - blockers.push(TableCatalogBackingMigrationBlocker::WarehouseIndexBackfillRequired); - recommended_actions.push(TableCatalogBackingMigrationAction::BackfillWarehouseIndex); - } - if duplicate_warehouse_prefix_count > 0 { - blockers.push(TableCatalogBackingMigrationBlocker::DuplicateWarehousePrefix); - recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateWarehousePrefixes); - } - - let mut status = if manual_review_count > 0 || duplicate_warehouse_prefix_count > 0 { - TableCatalogBackingMigrationStatus::ManualReviewRequired - } else if recovery_required_count > 0 || !warehouse_index_ready { - TableCatalogBackingMigrationStatus::RecoveryRequired - } else { - TableCatalogBackingMigrationStatus::ReadyToSnapshot - }; - - let strong_store = StrongTableCatalogStore::new(self.backend.clone()); - let migration_fence = self.read_backing_migration_fence(table_bucket).await?.map(|(fence, _)| fence); - let object_backed_writes_fenced = migration_fence.is_some(); - if status == TableCatalogBackingMigrationStatus::ReadyToSnapshot - && let Some(fence) = migration_fence.as_ref() - && fence.status == TableCatalogBackingMigrationFenceStatus::Materialized - && let Some(source_fingerprint) = fence.source_fingerprint.as_deref() - { - if strong_store.bucket_snapshot_fingerprint(table_bucket).await?.as_deref() == Some(source_fingerprint) { - status = TableCatalogBackingMigrationStatus::SnapshotMaterialized; - } else { - status = TableCatalogBackingMigrationStatus::ManualReviewRequired; - blockers.push(TableCatalogBackingMigrationBlocker::DurableStrongSnapshotChanged); - recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDurableStrongSnapshot); - } - } - - let ready_to_enable_durable_strong = status == TableCatalogBackingMigrationStatus::SnapshotMaterialized - && self.all_table_buckets_materialized(&strong_store).await?; - if status == TableCatalogBackingMigrationStatus::ReadyToSnapshot { - recommended_actions.extend([ - TableCatalogBackingMigrationAction::SnapshotObjectBackedCatalog, - TableCatalogBackingMigrationAction::KeepObjectBackedRollbackConfig, - ]); - } else if status == TableCatalogBackingMigrationStatus::SnapshotMaterialized { - recommended_actions.push(TableCatalogBackingMigrationAction::VerifyDurableStrongSnapshot); - if ready_to_enable_durable_strong { - recommended_actions.push(TableCatalogBackingMigrationAction::EnableDurableStrongBacking); - } else { - recommended_actions.push(TableCatalogBackingMigrationAction::SnapshotRemainingTableBuckets); - } - recommended_actions.push(TableCatalogBackingMigrationAction::KeepObjectBackedRollbackConfig); - } - - Ok(TableCatalogBackingMigrationDryRunReport { - table_bucket: table_bucket.to_string(), - source_kind: TableCatalogBackingKind::ObjectBacked, - target_kind: TableCatalogBackingKind::StrongKvWal, - status, - namespace_count: namespaces.len(), - table_count, - view_count, - commit_log_count, - idempotency_index_count, - warehouse_prefix_count: warehouse_prefix_owners.len(), - warehouse_index_ready, - object_backed_writes_fenced, - ready_to_enable_durable_strong, - blockers, - recommended_actions, - rollback: TableCatalogBackingRollbackPlan { - backing_config_key: ENV_TABLE_CATALOG_BACKING, - current_backing_value: TABLE_CATALOG_BACKING_DURABLE_STRONG, - rollback_backing_value: TABLE_CATALOG_BACKING_OBJECT, - preserves_object_backed_catalog: true, - requires_operator_restart: true, - }, - }) - } - - pub(crate) async fn materialize_durable_strong_backing_migration( - &self, - table_bucket: &str, - ) -> TableCatalogStoreResult { - let fence_path = self.paths.backing_migration_fence_path(table_bucket); - let fence_lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); - let global_fence_path = self.paths.backing_migration_global_fence_path(); - let global_fence_lock_path = self.paths.backing_migration_global_fence_lock_path(); - if self.read_backing_migration_fence(table_bucket).await?.is_none() { - let preflight = self.plan_durable_strong_backing_migration(table_bucket).await?; - if preflight.status != TableCatalogBackingMigrationStatus::ReadyToSnapshot { - return Err(TableCatalogStoreError::Conflict(format!( - "table bucket {table_bucket} is not ready for durable strong snapshot materialization" - ))); - } - } - - let _global_fence_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &global_fence_lock_path) - .await?; - let _fence_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &fence_lock_path) - .await?; - let existing_fence = self.read_backing_migration_fence(table_bucket).await?; - let strong_store = StrongTableCatalogStore::new(self.backend.clone()); - if let Some((fence, _)) = existing_fence.as_ref() - && (fence.version != TABLE_CATALOG_MIGRATION_VERSION || fence.table_bucket != table_bucket) - { - return Err(TableCatalogStoreError::Invalid(format!( - "invalid durable strong migration fence for table bucket {table_bucket}" - ))); - } - - if !self.warehouse_index_ready(table_bucket).await? { - return Err(TableCatalogStoreError::Conflict(format!( - "table bucket {table_bucket} warehouse index must be backfilled before durable strong migration" - ))); - } - - let mut source_guards = Vec::new(); - let source = self - .collect_bucket_snapshot_with_locks(table_bucket, &mut source_guards) - .await?; - let source_fingerprint = table_catalog_bucket_snapshot_fingerprint(&source)?; - if let Some((fence, _)) = existing_fence.as_ref() - && fence.status == TableCatalogBackingMigrationFenceStatus::Materialized - && fence.source_fingerprint.as_deref() != Some(source_fingerprint.as_str()) - { - return Err(TableCatalogStoreError::Conflict(format!( - "object-backed catalog state no longer matches the materialized snapshot for table bucket {table_bucket}" - ))); - } - - self.ensure_global_backing_migration_fence(&global_fence_path).await?; - let (migration_id, target_bucket_existed) = if let Some((fence, _)) = existing_fence.as_ref() { - (fence.migration_id.clone(), fence.target_bucket_existed) - } else { - let target_bucket_existed = strong_store.bucket_snapshot_fingerprint(table_bucket).await?.is_some(); - let fence = TableCatalogBackingMigrationFence { - version: TABLE_CATALOG_MIGRATION_VERSION, - table_bucket: table_bucket.to_string(), - migration_id: Uuid::new_v4().to_string(), - status: TableCatalogBackingMigrationFenceStatus::Preparing, - target_bucket_existed, - source_fingerprint: None, - target_snapshot_etag: None, - }; - self.write_entry(self.catalog_bucket(), &fence_path, &fence, TableCatalogPutPrecondition::IfAbsent) - .await?; - (fence.migration_id, target_bucket_existed) - }; - - let (target_snapshot_etag, created) = strong_store.materialize_bucket_snapshot(source.clone()).await?; - let completed_fence = TableCatalogBackingMigrationFence { - version: TABLE_CATALOG_MIGRATION_VERSION, - table_bucket: table_bucket.to_string(), - migration_id, - status: TableCatalogBackingMigrationFenceStatus::Materialized, - target_bucket_existed, - source_fingerprint: Some(source_fingerprint.clone()), - target_snapshot_etag: Some(target_snapshot_etag.clone()), - }; - self.write_entry(self.catalog_bucket(), &fence_path, &completed_fence, TableCatalogPutPrecondition::Any) - .await?; - - drop(source_guards); - drop(_fence_guard); - let ready_to_enable_durable_strong = self.all_table_buckets_materialized(&strong_store).await?; - Ok(TableCatalogBackingMigrationExecutionReport { - table_bucket: table_bucket.to_string(), - source_kind: TableCatalogBackingKind::ObjectBacked, - target_kind: TableCatalogBackingKind::StrongKvWal, - status: if created { - TableCatalogBackingMigrationExecutionStatus::SnapshotMaterialized - } else { - TableCatalogBackingMigrationExecutionStatus::SnapshotAlreadyMaterialized - }, - namespace_count: source.namespaces.len(), - table_count: source.tables.len(), - view_count: source.views.len(), - commit_log_count: source.commits.len(), - idempotency_index_count: source.idempotency.len(), - source_fingerprint, - target_snapshot_etag, - object_backed_writes_fenced: true, - ready_to_enable_durable_strong, - }) - } - - pub(crate) async fn cancel_durable_strong_backing_migration( - &self, - table_bucket: &str, - ) -> TableCatalogStoreResult { - let fence_path = self.paths.backing_migration_fence_path(table_bucket); - let fence_lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); - let global_fence_path = self.paths.backing_migration_global_fence_path(); - let global_fence_lock_path = self.paths.backing_migration_global_fence_lock_path(); - let _global_fence_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &global_fence_lock_path) - .await?; - let _fence_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &fence_lock_path) - .await?; - let Some((fence, _)) = self.read_backing_migration_fence(table_bucket).await? else { - self.clear_global_backing_migration_fence_if_unused(&global_fence_path) - .await?; - return Ok(TableCatalogBackingMigrationCancelReport { - table_bucket: table_bucket.to_string(), - status: TableCatalogBackingMigrationCancelStatus::NoMigrationFence, - object_backed_writes_fenced: false, - }); - }; - self.ensure_global_backing_migration_fence(&global_fence_path).await?; - if fence.version != TABLE_CATALOG_MIGRATION_VERSION || fence.table_bucket != table_bucket { - return Err(TableCatalogStoreError::Invalid(format!( - "invalid durable strong migration fence for table bucket {table_bucket}" - ))); - } - - let mut source_guards = Vec::new(); - let source = self - .collect_bucket_snapshot_with_locks(table_bucket, &mut source_guards) - .await?; - let source_fingerprint = table_catalog_bucket_snapshot_fingerprint(&source)?; - if fence.status == TableCatalogBackingMigrationFenceStatus::Materialized - && fence.source_fingerprint.as_deref() != Some(source_fingerprint.as_str()) - { - return Err(TableCatalogStoreError::Conflict(format!( - "object-backed catalog state changed after materializing table bucket {table_bucket}" - ))); - } - - let strong_store = StrongTableCatalogStore::new(self.backend.clone()); - if fence.status == TableCatalogBackingMigrationFenceStatus::Materialized - && strong_store.bucket_snapshot_fingerprint(table_bucket).await?.as_deref() != Some(&source_fingerprint) - { - return Err(TableCatalogStoreError::Conflict(format!( - "durable strong catalog state changed after materializing table bucket {table_bucket}" - ))); - } - if !fence.target_bucket_existed { - strong_store - .remove_bucket_snapshot_if_unchanged(table_bucket, &source_fingerprint) - .await?; - } - self.backend.delete_object(self.catalog_bucket(), &fence_path).await?; - self.clear_global_backing_migration_fence_if_unused(&global_fence_path) - .await?; - Ok(TableCatalogBackingMigrationCancelReport { - table_bucket: table_bucket.to_string(), - status: TableCatalogBackingMigrationCancelStatus::FenceReleased, - object_backed_writes_fenced: false, - }) - } - - async fn all_table_buckets_materialized(&self, strong_store: &StrongTableCatalogStore) -> TableCatalogStoreResult { - if self - .read_entry::( - self.catalog_bucket(), - &self.paths.backing_migration_global_fence_path(), - ) - .await? - .is_none() - { - return Ok(false); - } - let table_bucket_objects = self - .backend - .list_objects(self.catalog_bucket(), &self.paths.table_bucket_entries_prefix()) - .await?; - for table_bucket_object in table_bucket_objects - .iter() - .filter(|object| object.ends_with(TABLE_BUCKET_ENTRY_FILE)) - { - let Some((entry, _)) = self - .read_entry::(self.catalog_bucket(), table_bucket_object) - .await? - else { - return Ok(false); - }; - let Some((fence, _)) = self.read_backing_migration_fence(&entry.table_bucket).await? else { - return Ok(false); - }; - if fence.status != TableCatalogBackingMigrationFenceStatus::Materialized { - return Ok(false); - } - let Some(source_fingerprint) = fence.source_fingerprint.as_deref() else { - return Ok(false); - }; - if strong_store - .bucket_snapshot_fingerprint(&entry.table_bucket) - .await? - .as_deref() - != Some(source_fingerprint) - { - return Ok(false); - } - } - Ok(true) - } - - pub(crate) async fn diagnose_table_catalog( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - retain_recent_metadata_files: usize, - ) -> TableCatalogStoreResult { - let parsed_namespace = parse_namespace_for_store(namespace)?; - let parsed_table = parse_table_for_store(table)?; - let catalog = self.export_table_catalog_entry(table_bucket, namespace, table).await?; - let current_metadata_location = catalog.table.metadata_location.clone(); - - let mut retained = BTreeSet::new(); - let mut current_metadata_for_refs = None; - let current_metadata_status = - if is_valid_table_metadata_location(&parsed_namespace, &parsed_table, ¤t_metadata_location) { - retained.insert(current_metadata_location.clone()); - match self.backend.read_object(table_bucket, ¤t_metadata_location).await? { - Some(current_metadata_object) => { - match serde_json::from_slice::(¤t_metadata_object.data) { - Ok(current_metadata) if current_metadata.is_object() => { - retained.extend(metadata_log_locations(¤t_metadata, &parsed_namespace, &parsed_table)); - current_metadata_for_refs = Some(current_metadata); - TableMetadataPointerStatus::Valid - } - Ok(_) | Err(_) => TableMetadataPointerStatus::InvalidJson, - } - } - None => TableMetadataPointerStatus::MissingObject, - } - } else { - TableMetadataPointerStatus::InvalidLocation - }; - - let mut metadata_locations = Vec::new(); - let metadata_prefix = format!("{}/", default_table_metadata_dir_path(&parsed_namespace, &parsed_table)); - for object in self.backend.list_objects(table_bucket, &metadata_prefix).await? { - if let Some(metadata_location) = metadata_location_from_metadata_file_path(&parsed_namespace, &parsed_table, &object) - { - metadata_locations.push(metadata_location); - } - } - metadata_locations.sort(); - metadata_locations.dedup(); - - for metadata_location in metadata_locations.iter().rev().take(retain_recent_metadata_files) { - retained.insert(metadata_location.clone()); - } - if let Some(current_metadata) = current_metadata_for_refs.as_ref() { - retained.extend( - metadata_locations_for_protected_snapshot_refs( - &self.backend, - table_bucket, - &parsed_namespace, - &parsed_table, - current_metadata, - &metadata_locations, - ) - .await?, - ); - } - - let orphan_metadata_candidate_locations = metadata_locations - .into_iter() - .filter(|metadata_location| !retained.contains(metadata_location)) - .collect(); - - let commit_recovery = self.plan_table_commit_recovery(table_bucket, namespace, table).await?; - let (recovery_status, recommended_actions) = table_catalog_recovery_summary(¤t_metadata_status, &commit_recovery); - let backing_manifest = - table_catalog_backing_manifest(&self.paths, &parsed_namespace, &parsed_table, &catalog.table, &commit_recovery); - - Ok(TableCatalogDiagnosticsReport { - catalog, - current_metadata_status, - recovery_status, - recommended_actions, - commit_recovery, - backing_manifest, - orphan_metadata_candidate_locations, - }) - } - - pub(crate) async fn plan_table_metadata_maintenance( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - retain_recent_metadata_files: usize, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { - return Err(TableCatalogStoreError::Invalid( - "current metadata location must be inside the table metadata directory".to_string(), - )); - } - - let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "current metadata object {}", - entry.metadata_location - ))); - }; - let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { - TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) - })?; - if !current_metadata.is_object() { - return Err(TableCatalogStoreError::Invalid(format!( - "current metadata {} must be a JSON object", - entry.metadata_location - ))); - } - - let mut retained = BTreeSet::new(); - let mut maintenance_reasons = BTreeMap::>::new(); - for metadata_location in metadata_log_locations(¤t_metadata, &namespace, &table) { - retained.insert(metadata_location.clone()); - insert_metadata_maintenance_reason( - &mut maintenance_reasons, - metadata_location, - TableMetadataMaintenanceReason::MetadataLog, - ); - } - retained.insert(entry.metadata_location.clone()); - insert_metadata_maintenance_reason( - &mut maintenance_reasons, - entry.metadata_location.clone(), - TableMetadataMaintenanceReason::CurrentMetadata, - ); - - let mut metadata_locations = Vec::new(); - let metadata_prefix = format!("{}/", default_table_metadata_dir_path(&namespace, &table)); - for object in self.backend.list_objects(table_bucket, &metadata_prefix).await? { - if let Some(metadata_location) = metadata_location_from_metadata_file_path(&namespace, &table, &object) { - metadata_locations.push(metadata_location); - } - } - metadata_locations.sort(); - metadata_locations.dedup(); - let planned_metadata_file_count = metadata_locations.len(); - - for metadata_location in metadata_locations.iter().rev().take(retain_recent_metadata_files) { - retained.insert(metadata_location.clone()); - if metadata_location != &entry.metadata_location { - insert_metadata_maintenance_reason( - &mut maintenance_reasons, - metadata_location.clone(), - TableMetadataMaintenanceReason::RecentMetadata, - ); - } - } - for metadata_location in metadata_locations_for_protected_snapshot_refs( - &self.backend, - table_bucket, - &namespace, - &table, - ¤t_metadata, - &metadata_locations, - ) - .await? - { - retained.insert(metadata_location.clone()); - insert_metadata_maintenance_reason( - &mut maintenance_reasons, - metadata_location, - TableMetadataMaintenanceReason::ProtectedSnapshotRef, - ); - } - - let cleanup_candidate_locations = metadata_locations - .iter() - .filter(|metadata_location| !retained.contains(metadata_location.as_str())) - .cloned() - .collect::>(); - - let now = OffsetDateTime::now_utc(); - let mut deletable_metadata_locations = Vec::new(); - for metadata_location in &cleanup_candidate_locations { - insert_metadata_maintenance_reason( - &mut maintenance_reasons, - metadata_location.clone(), - TableMetadataMaintenanceReason::NoCurrentReachability, - ); - let Some(candidate_object) = self.backend.read_object(table_bucket, metadata_location).await? else { - insert_metadata_maintenance_reason( - &mut maintenance_reasons, - metadata_location.clone(), - TableMetadataMaintenanceReason::SafetyWindowPending, - ); - continue; - }; - if metadata_candidate_is_past_safety_window(candidate_object.mod_time, now) { - deletable_metadata_locations.push(metadata_location.clone()); - insert_metadata_maintenance_reason( - &mut maintenance_reasons, - metadata_location.clone(), - TableMetadataMaintenanceReason::SafetyWindowSatisfied, - ); - } else { - insert_metadata_maintenance_reason( - &mut maintenance_reasons, - metadata_location.clone(), - TableMetadataMaintenanceReason::SafetyWindowPending, - ); - } - } - let warehouse_object_prefix = table_warehouse_object_prefix(&entry).ok(); - let current_metadata_location = entry.metadata_location; - let retained_metadata_locations = retained.into_iter().collect::>(); - let object_reports = metadata_maintenance_object_reports(maintenance_reasons); - let referenced_object_reports = metadata_maintenance_referenced_object_reports( - &self.backend, - table_bucket, - &namespace, - &table, - warehouse_object_prefix.as_deref(), - ¤t_metadata, - &retained_metadata_locations, - ) - .await?; - let reachability_graph = - metadata_maintenance_reachability_graph_report(planned_metadata_file_count, &referenced_object_reports); - let (planned_object_file_count, cleanup_object_candidate_locations, deletable_object_locations, object_cleanup_reports) = - metadata_maintenance_object_cleanup_reports( - &self.backend, - table_bucket, - &namespace, - &table, - warehouse_object_prefix.as_deref(), - &referenced_object_reports, - now, - ) - .await?; - - let mut report = table_maintenance_report_with_recommended_actions(TableMetadataMaintenanceReport { - job: TableMetadataMaintenanceJob { - job_id: Uuid::new_v4().to_string(), - table_bucket: table_bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: entry.table_id, - operation: TableMetadataMaintenanceOperation::DryRun, - status: TableMetadataMaintenanceJobStatus::Successful, - failure_reason: None, - recommended_actions: Vec::new(), - config_source: TableMaintenanceConfigSource::Default, - scheduler_id: None, - scheduler_lease_id: String::new(), - scheduled_at: None, - worker_id: None, - lease_id: String::new(), - attempt: 0, - max_retry_attempts: 0, - next_retry_after: None, - quarantine_enabled: false, - quarantine_retention_seconds: 0, - heartbeat_at: None, - started_at: None, - finished_at: None, - current_metadata_location: current_metadata_location.clone(), - current_generation: entry.generation, - retain_recent_metadata_files, - safety_window_seconds: TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS, - cleanup_watermark_unix_seconds: (now - Duration::seconds(TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS)) - .unix_timestamp(), - planned_metadata_file_count, - retained_metadata_file_count: retained_metadata_locations.len(), - cleanup_candidate_count: cleanup_candidate_locations.len(), - deletable_metadata_file_count: deletable_metadata_locations.len(), - deleted_metadata_file_count: 0, - planned_object_file_count, - cleanup_candidate_object_count: cleanup_object_candidate_locations.len(), - deletable_object_count: deletable_object_locations.len(), - deleted_object_count: 0, - quarantined_object_count: 0, - }, - current_metadata_location, - retained_metadata_locations, - cleanup_candidate_locations, - deletable_metadata_locations, - cleanup_object_candidate_locations, - deletable_object_locations, - object_reports, - object_cleanup_reports, - referenced_object_reports, - reachability_graph, - snapshot_expiration: None, - compaction: None, - audit_events: Vec::new(), - }); - push_table_maintenance_audit_event( - &mut report, - now, - TableMaintenanceAuditActor::Scheduler, - TableMaintenanceAuditAction::Planned, - None, - None, - None, - ); - Ok(report) - } - - pub(crate) async fn delete_table_metadata_maintenance_candidates( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - retain_recent_metadata_files: usize, - ) -> TableCatalogStoreResult { - let report = self - .plan_table_metadata_maintenance(table_bucket, namespace, table, retain_recent_metadata_files) - .await?; - self.delete_table_metadata_maintenance_report(table_bucket, namespace, table, report) - .await - } - - pub(crate) async fn run_table_metadata_maintenance( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - delete: bool, - worker_id: Option, - ) -> TableCatalogStoreResult { - let effective = self - .get_effective_table_maintenance_config(table_bucket, namespace, table) - .await?; - self.run_table_metadata_maintenance_with_config(table_bucket, namespace, table, delete, worker_id, effective) - .await - } - - pub(crate) async fn run_table_metadata_maintenance_with_retention( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - delete: bool, - worker_id: Option, - retain_recent_metadata_files: usize, - ) -> TableCatalogStoreResult { - let mut effective = self - .get_effective_table_maintenance_config(table_bucket, namespace, table) - .await?; - effective.config.retain_recent_metadata_files = retain_recent_metadata_files; - self.run_table_metadata_maintenance_with_config(table_bucket, namespace, table, delete, worker_id, effective) - .await - } - - async fn run_table_metadata_maintenance_with_config( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - delete: bool, - worker_id: Option, - effective: TableMaintenanceEffectiveConfig, - ) -> TableCatalogStoreResult { - let mut report = self - .plan_table_metadata_maintenance(table_bucket, namespace, table, effective.config.retain_recent_metadata_files) - .await?; - - let started_at_time = OffsetDateTime::now_utc(); - let started_at = maintenance_timestamp(started_at_time); - report.job.operation = if delete { - TableMetadataMaintenanceOperation::Delete - } else { - TableMetadataMaintenanceOperation::DryRun - }; - report.job.status = TableMetadataMaintenanceJobStatus::Running; - report.job.failure_reason = None; - report.job.config_source = effective.source; - report.job.worker_id = worker_id; - report.job.lease_id = Uuid::new_v4().to_string(); - report.job.attempt = 1; - report.job.max_retry_attempts = effective.config.max_retry_attempts; - report.job.next_retry_after = None; - report.job.quarantine_enabled = effective.config.quarantine_enabled; - report.job.quarantine_retention_seconds = effective.config.quarantine_retention_seconds; - report.job.heartbeat_at = Some(started_at.clone()); - report.job.started_at = Some(started_at); - report.job.finished_at = None; - refresh_table_maintenance_report_recommended_actions(&mut report); - push_table_maintenance_audit_event( - &mut report, - started_at_time, - TableMaintenanceAuditActor::Worker, - TableMaintenanceAuditAction::WorkerStarted, - None, - Some(TableMetadataMaintenanceJobStatus::Successful), - Some(0), - ); - self.put_table_metadata_maintenance_report(&report).await?; - - self.finish_table_metadata_maintenance_run(table_bucket, namespace, table, delete, &effective, report) - .await - } - - async fn finish_table_metadata_maintenance_run( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - delete: bool, - effective: &TableMaintenanceEffectiveConfig, - mut report: TableMetadataMaintenanceReport, - ) -> TableCatalogStoreResult { - if delete && !effective.config.delete_enabled { - let finished_at = OffsetDateTime::now_utc(); - let before_status = Some(report.job.status.clone()); - let before_quarantined_object_count = Some(report.job.quarantined_object_count); - report.job.status = TableMetadataMaintenanceJobStatus::Failed; - report.job.failure_reason = Some(TABLE_MAINTENANCE_DELETE_DISABLED_REASON.to_string()); - apply_maintenance_retry_after(&mut report.job, &effective.config, finished_at); - report.job.finished_at = Some(maintenance_timestamp(finished_at)); - refresh_table_maintenance_report_recommended_actions(&mut report); - push_table_maintenance_audit_event( - &mut report, - finished_at, - TableMaintenanceAuditActor::Worker, - TableMaintenanceAuditAction::WorkerFailed, - Some(TABLE_MAINTENANCE_DELETE_DISABLED_REASON.to_string()), - before_status, - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&report).await?; - return Ok(report); - } - - if delete { - let running_report = report.clone(); - let mut deleted = match self - .delete_table_metadata_maintenance_report(table_bucket, namespace, table, report) - .await - { - Ok(report) => report, - Err(err) => { - let finished_at = OffsetDateTime::now_utc(); - let mut failed = running_report; - let before_status = Some(failed.job.status.clone()); - let before_quarantined_object_count = Some(failed.job.quarantined_object_count); - let reason = err.to_string(); - failed.job.status = TableMetadataMaintenanceJobStatus::Failed; - failed.job.failure_reason = Some(reason.clone()); - apply_maintenance_retry_after(&mut failed.job, &effective.config, finished_at); - failed.job.finished_at = Some(maintenance_timestamp(finished_at)); - refresh_table_maintenance_report_recommended_actions(&mut failed); - push_table_maintenance_audit_event( - &mut failed, - finished_at, - TableMaintenanceAuditActor::Worker, - TableMaintenanceAuditAction::WorkerFailed, - Some(reason), - before_status, - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&failed).await?; - return Err(err); - } - }; - let finished_at = OffsetDateTime::now_utc(); - let before_status = Some(TableMetadataMaintenanceJobStatus::Running); - let before_quarantined_object_count = Some(0); - deleted.job.finished_at = Some(maintenance_timestamp(finished_at)); - refresh_table_maintenance_report_recommended_actions(&mut deleted); - push_table_maintenance_audit_event( - &mut deleted, - finished_at, - TableMaintenanceAuditActor::Worker, - TableMaintenanceAuditAction::WorkerSucceeded, - None, - before_status, - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&deleted).await?; - return Ok(deleted); - } - - let finished_at = OffsetDateTime::now_utc(); - let before_status = Some(report.job.status.clone()); - let before_quarantined_object_count = Some(report.job.quarantined_object_count); - report.job.status = TableMetadataMaintenanceJobStatus::Successful; - report.job.finished_at = Some(maintenance_timestamp(finished_at)); - refresh_table_maintenance_report_recommended_actions(&mut report); - push_table_maintenance_audit_event( - &mut report, - finished_at, - TableMaintenanceAuditActor::Worker, - TableMaintenanceAuditAction::WorkerSucceeded, - None, - before_status, - before_quarantined_object_count, - ); - self.put_table_metadata_maintenance_report(&report).await?; - Ok(report) - } - - async fn delete_table_metadata_maintenance_report( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - report: TableMetadataMaintenanceReport, - ) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - if !is_valid_table_metadata_location(&namespace, &table, &report.current_metadata_location) { - return Err(TableCatalogStoreError::Invalid( - "maintenance report current metadata location must be inside the table metadata directory".to_string(), - )); - } - - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - if entry.metadata_location != report.current_metadata_location { - return Err(TableCatalogStoreError::Conflict( - "current metadata location changed before maintenance delete".to_string(), - )); - } - let warehouse_object_prefix = table_warehouse_object_prefix(&entry).ok(); - - let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "current metadata object {}", - entry.metadata_location - ))); - }; - let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { - TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) - })?; - if !current_metadata.is_object() { - return Err(TableCatalogStoreError::Invalid(format!( - "current metadata {} must be a JSON object", - entry.metadata_location - ))); - } - - let mut protected = metadata_log_locations(¤t_metadata, &namespace, &table); - protected.insert(entry.metadata_location.clone()); - protected.extend(report.retained_metadata_locations.iter().cloned()); - protected.extend( - metadata_locations_for_protected_snapshot_refs( - &self.backend, - table_bucket, - &namespace, - &table, - ¤t_metadata, - &report.cleanup_candidate_locations, - ) - .await?, - ); - - let cleanup_candidate_count = report.cleanup_candidate_locations.len(); - let planned_deletable_locations = report.deletable_metadata_locations.iter().cloned().collect::>(); - let mut cleanup_candidate_locations = BTreeSet::new(); - let now = OffsetDateTime::now_utc(); - for metadata_location in &report.cleanup_candidate_locations { - if !is_valid_table_metadata_location(&namespace, &table, metadata_location) { - return Err(TableCatalogStoreError::Invalid(format!( - "cleanup candidate {metadata_location} must be inside the table metadata directory" - ))); - } - if protected.contains(metadata_location.as_str()) { - return Err(TableCatalogStoreError::Conflict(format!( - "cleanup candidate {metadata_location} is retained by current metadata" - ))); - } - let Some(candidate_object) = self.backend.read_object(table_bucket, metadata_location).await? else { - continue; - }; - if !planned_deletable_locations.contains(metadata_location.as_str()) { - continue; - } - if !metadata_candidate_is_past_safety_window(candidate_object.mod_time, now) { - continue; - } - cleanup_candidate_locations.insert(metadata_location.clone()); - } - - let cleanup_candidate_locations = cleanup_candidate_locations.into_iter().collect::>(); - let deleted_locations = cleanup_candidate_locations.iter().cloned().collect::>(); - for metadata_location in &cleanup_candidate_locations { - self.backend.delete_object(table_bucket, metadata_location).await?; - } - - let referenced_object_reports = metadata_maintenance_referenced_object_reports( - &self.backend, - table_bucket, - &namespace, - &table, - warehouse_object_prefix.as_deref(), - ¤t_metadata, - &report.retained_metadata_locations, - ) - .await?; - let referenced_object_locations = if referenced_object_reports - .iter() - .any(|report| report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired) - { - BTreeSet::new() - } else { - referenced_object_reports - .iter() - .filter_map(|report| table_catalog_object_key_from_location(table_bucket, &report.object_location)) - .collect::>() - }; - let planned_deletable_object_locations = report.deletable_object_locations.iter().cloned().collect::>(); - let mut cleanup_object_candidate_locations = BTreeSet::new(); - if !referenced_object_reports - .iter() - .any(|report| report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired) - { - for object_location in &report.cleanup_object_candidate_locations { - if table_maintenance_object_kind(&namespace, &table, warehouse_object_prefix.as_deref(), object_location) - .is_none() - { - return Err(TableCatalogStoreError::Invalid(format!( - "cleanup object candidate {object_location} must be inside table metadata, data, or delete directories" - ))); - } - if referenced_object_locations.contains(object_location.as_str()) { - return Err(TableCatalogStoreError::Conflict(format!( - "cleanup object candidate {object_location} is retained by current metadata" - ))); - } - let Some(candidate_object) = self.backend.read_object(table_bucket, object_location).await? else { - continue; - }; - if !planned_deletable_object_locations.contains(object_location.as_str()) { - continue; - } - if !metadata_candidate_is_past_safety_window(candidate_object.mod_time, now) { - continue; - } - cleanup_object_candidate_locations.insert(object_location.clone()); - } - } - - let cleanup_object_candidate_locations = cleanup_object_candidate_locations.into_iter().collect::>(); - let deleted_object_locations = cleanup_object_candidate_locations.iter().cloned().collect::>(); - for object_location in &cleanup_object_candidate_locations { - self.backend.delete_object(table_bucket, object_location).await?; - } - - let retained_metadata_locations = protected.into_iter().collect::>(); - let mut job = report.job; - job.operation = TableMetadataMaintenanceOperation::Delete; - job.status = TableMetadataMaintenanceJobStatus::Successful; - job.failure_reason = None; - job.retained_metadata_file_count = retained_metadata_locations.len(); - job.cleanup_candidate_count = cleanup_candidate_count; - job.deletable_metadata_file_count = planned_deletable_locations.len(); - job.deleted_metadata_file_count = cleanup_candidate_locations.len(); - job.cleanup_candidate_object_count = report.cleanup_object_candidate_locations.len(); - job.deletable_object_count = planned_deletable_object_locations.len(); - job.deleted_object_count = cleanup_object_candidate_locations.len(); - let mut object_reports = report.object_reports; - mark_deleted_metadata_object_reports(&mut object_reports, &deleted_locations); - let mut object_cleanup_reports = report.object_cleanup_reports; - mark_deleted_object_cleanup_reports(&mut object_cleanup_reports, &deleted_object_locations); - - Ok(table_maintenance_report_with_recommended_actions(TableMetadataMaintenanceReport { - job, - current_metadata_location: entry.metadata_location, - retained_metadata_locations, - cleanup_candidate_locations: cleanup_candidate_locations.clone(), - deletable_metadata_locations: cleanup_candidate_locations, - cleanup_object_candidate_locations: cleanup_object_candidate_locations.clone(), - deletable_object_locations: cleanup_object_candidate_locations, - object_reports, - object_cleanup_reports, - referenced_object_reports, - reachability_graph: report.reachability_graph, - snapshot_expiration: report.snapshot_expiration, - compaction: report.compaction, - audit_events: report.audit_events, - })) - } -} - -#[async_trait::async_trait] -impl TableCatalogStore for ObjectTableCatalogStore -where - B: TableCatalogObjectBackend, -{ - async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult> { - self.read_entry::(self.catalog_bucket(), &self.paths.table_bucket_entry_path(table_bucket)) - .await - .map(|entry| entry.map(|(bucket, _)| bucket)) - } - - async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { - validate_catalog_entry_version("table bucket", entry.version)?; - if entry.table_bucket.is_empty() { - return Err(TableCatalogStoreError::Invalid("table bucket name cannot be empty".to_string())); - } - if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE { - return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string())); - } - let _registry_guard = self.acquire_table_bucket_registry_write_permit().await?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; - let object = self.paths.table_bucket_entry_path(&entry.table_bucket); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; - self.write_entry_unlocked(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::Any) - .await - } - - async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> { - validate_catalog_entry_version("namespace", entry.version)?; - self.require_table_bucket(&entry.table_bucket).await?; - let namespace = parse_namespace_for_store(&entry.namespace)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; - let bucket_path = self.paths.table_bucket_entry_path(&entry.table_bucket); - let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?; - let object = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); - self.write_entry(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::IfAbsent) - .await - } - - async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult> { - let mut entries = Vec::new(); - for object in self - .backend - .list_objects(self.catalog_bucket(), &self.paths.namespace_entries_prefix(table_bucket)) - .await? - { - if !object.ends_with(NAMESPACE_ENTRY_FILE) { - continue; - } - if let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? { - entries.push(entry); - } - } - entries.sort_by(|left, right| left.namespace.cmp(&right.namespace)); - Ok(entries) - } - - async fn list_namespaces_page( - &self, - table_bucket: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - self.list_entry_page(&self.paths.namespace_entries_prefix(table_bucket), NAMESPACE_ENTRY_FILE, cursor, limit) - .await - } - - async fn get_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - self.read_entry::(self.catalog_bucket(), &self.paths.namespace_entry_path(table_bucket, &namespace)) - .await - .map(|entry| entry.map(|(namespace, _)| namespace)) - } - - async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()> { - let namespace = parse_namespace_for_store(namespace)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; - let bucket_path = self.paths.table_bucket_entry_path(table_bucket); - let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?; - let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); - // Match create_namespace and migration lock order while draining table/view creation. - let _namespace_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &namespace_path) - .await?; - if self - .read_entry_unlocked::(self.catalog_bucket(), &namespace_path) - .await? - .is_none() - { - return Err(TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - table_bucket, - namespace.public_name() - ))); - } - if !self.list_tables(table_bucket, &namespace.public_name()).await?.is_empty() { - return Err(TableCatalogStoreError::Conflict(format!( - "namespace {}/{} is not empty", - table_bucket, - namespace.public_name() - ))); - } - if !self.list_views(table_bucket, &namespace.public_name()).await?.is_empty() { - return Err(TableCatalogStoreError::Conflict(format!( - "namespace {}/{} is not empty", - table_bucket, - namespace.public_name() - ))); - } - self.backend - .delete_object_unlocked(self.catalog_bucket(), &namespace_path) - .await - } - - async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { - self.write_table_entry(entry, TableCatalogPutPrecondition::IfAbsent).await - } - - async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { - self.write_table_entry(entry, TableCatalogPutPrecondition::IfAbsent).await - } - - async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - let mut entries = Vec::new(); - for object in self - .backend - .list_objects(self.catalog_bucket(), &self.paths.table_entries_prefix(table_bucket, &namespace)) - .await? - { - if !object.ends_with(TABLE_ENTRY_FILE) { - continue; - } - if let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? { - entries.push(entry); - } - } - entries.sort_by(|left, right| left.table.cmp(&right.table)); - Ok(entries) - } - - async fn list_tables_page( - &self, - table_bucket: &str, - namespace: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - self.list_entry_page( - &self.paths.table_entries_prefix(table_bucket, &namespace), - TABLE_ENTRY_FILE, - cursor, - limit, - ) - .await - } - - async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - self.read_entry::(self.catalog_bucket(), &self.paths.table_entry_path(table_bucket, &namespace, &table)) - .await - .map(|entry| entry.map(|(table, _)| table)) - } - - async fn resolve_table_data_plane_resource( - &self, - table_bucket: &str, - object: &str, - ) -> TableCatalogStoreResult> { - if table_bucket.is_empty() || object.is_empty() { - return Ok(None); - } - let Some(table_bucket_entry) = self.get_table_bucket(table_bucket).await? else { - return Ok(None); - }; - if table_bucket_entry.state != TableCatalogEntryState::Active { - return Ok(None); - } - - if self.warehouse_index_ready(table_bucket).await? { - return self.resolve_table_data_plane_resource_from_index(table_bucket, object).await; - } - - match self.backfill_table_warehouse_index(table_bucket).await { - Ok(()) => self.resolve_table_data_plane_resource_from_index(table_bucket, object).await, - Err(err) => { - tracing::warn!( - table_bucket = %table_bucket, - error = %err, - "failed to backfill table warehouse index; falling back to catalog scan" - ); - scan_table_data_plane_resource_for_object(self, table_bucket, object).await - } - } - } - - async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { - let commit_started = Instant::now(); - record_table_commit_attempt(&request.operation); - let namespace = parse_namespace_for_store(&request.namespace)?; - let table = parse_table_for_store(&request.table)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; - let table_path = self.paths.table_entry_path(&request.table_bucket, &namespace, &table); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; - - let Some((current, current_etag)) = self - .read_table_with_etag_unlocked(&request.table_bucket, &namespace, &table) - .await? - else { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - request.table_bucket, request.namespace, request.table - ))), - ); - }; - - let commit_path = self - .paths - .commit_log_entry_path(&request.table_bucket, ¤t.table_id, &request.commit_id); - let existing_commit = self.read_commit_by_path(&commit_path).await?; - let idempotency_path = request.idempotency_key.as_deref().map(|idempotency_key| { - self.paths - .commit_idempotency_entry_path(&request.table_bucket, ¤t.table_id, idempotency_key) - }); - let existing_idempotency_commit = match idempotency_path.as_deref() { - Some(idempotency_path) => self.read_commit_by_path(idempotency_path).await?, - None => None, - }; - - if let Some(existing) = existing_commit.as_ref() { - if !commit_log_matches_request(existing, &request, ¤t.table_id) { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Conflict(format!( - "commit id already exists: {}", - request.commit_id - ))), - ); - } - if matches!(existing.status, CommitLogStatus::Committed) || table_matches_committed_log(¤t, existing) { - let mut committed = existing.clone(); - committed.status = CommitLogStatus::Committed; - let _ = self - .finalize_commit_log(&commit_path, idempotency_path.as_deref(), &committed) - .await; - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Ok(TableCommitResult { - table: current, - commit_log: committed, - }), - ); - } - if !matches!(existing.status, CommitLogStatus::Staged) || !table_matches_staged_base(¤t, existing) { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Conflict( - "existing commit record does not match current table state".to_string(), - )), - ); - } - } - if let Some(existing) = existing_idempotency_commit.as_ref() - && !commit_log_matches_request(existing, &request, ¤t.table_id) - { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Conflict("idempotency key already exists".to_string())), - ); - } - if existing_commit.is_none() && existing_idempotency_commit.is_some() { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Conflict( - "idempotency key exists without a recoverable commit record".to_string(), - )), - ); - } - - if current.version_token != request.expected_version_token { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Conflict( - "current table version token does not match expected token".to_string(), - )), - ); - } - if current.metadata_location != request.expected_metadata_location { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Conflict( - "current table metadata location does not match expected location".to_string(), - )), - ); - } - if !is_valid_table_metadata_location(&namespace, &table, &request.new_metadata_location) { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Invalid( - "new metadata location must be inside the table metadata directory".to_string(), - )), - ); - } - let Some(new_metadata_object) = self - .backend - .read_object(&request.table_bucket, &request.new_metadata_location) - .await? - else { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::NotFound(format!( - "new metadata object {}", - request.new_metadata_location - ))), - ); - }; - let next_warehouse_location = - table_metadata_warehouse_location(&request.table_bucket, &request.new_metadata_location, &new_metadata_object)?; - - let has_existing_commit = existing_commit.is_some(); - let mut staged_commit_log = existing_commit.unwrap_or_else(|| CommitLogEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - commit_id: request.commit_id.clone(), - idempotency_key: request.idempotency_key.clone(), - table_id: current.table_id.clone(), - operation: request.operation.clone(), - expected_version_token: request.expected_version_token.clone(), - new_version_token: format!("token-{}", Uuid::new_v4()), - previous_metadata_location: current.metadata_location.clone(), - new_metadata_location: request.new_metadata_location.clone(), - requirements: request.requirements.clone(), - status: CommitLogStatus::Staged, - writer: request.writer.clone(), - created_at: None, - updated_at: None, - }); - staged_commit_log.status = CommitLogStatus::Staged; - - let mut next = current.clone(); - next.metadata_location = staged_commit_log.new_metadata_location.clone(); - if let Some(warehouse_location) = next_warehouse_location { - next.warehouse_location = warehouse_location; - } - next.version_token = staged_commit_log.new_version_token.clone(); - next.generation = current.generation.saturating_add(1); - let reservation = self.reserve_table_warehouse_index(&next).await?; - - let staged_write_result = async { - if !has_existing_commit { - self.write_entry( - self.catalog_bucket(), - &commit_path, - &staged_commit_log, - TableCatalogPutPrecondition::IfAbsent, - ) - .await?; - } - if let Some(idempotency_path) = idempotency_path.as_deref() - && existing_idempotency_commit.is_none() - { - self.write_entry( - self.catalog_bucket(), - idempotency_path, - &staged_commit_log, - TableCatalogPutPrecondition::IfAbsent, - ) - .await?; - } - Ok(()) - } - .await; - if let Err(err) = staged_write_result { - self.delete_created_table_warehouse_index(&next, reservation, "commit staging failed") - .await; - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(err), - ); - } - - let cas_started = Instant::now(); - let cas_result = self - .write_entry_unlocked( - self.catalog_bucket(), - &table_path, - &next, - TableCatalogPutPrecondition::IfMatch(current_etag), - ) - .await; - record_table_commit_cas_result(&request.operation, cas_started, &cas_result); - if let Err(err) = cas_result { - self.delete_created_table_warehouse_index(&next, reservation, "table pointer CAS failed") - .await; - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(err), - ); - } - self.delete_table_warehouse_index_if_changed(¤t, &next).await; - - let mut commit_log = staged_commit_log; - commit_log.status = CommitLogStatus::Committed; - // After the table CAS succeeds, the staged record is the durable recovery source. - // A finalization failure must not turn an externally committed pointer into a failed commit response. - let _ = self - .finalize_commit_log(&commit_path, idempotency_path.as_deref(), &commit_log) - .await; - - table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Ok(TableCommitResult { table: next, commit_log }), - ) - } - - async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; - let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); - let _namespace_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &namespace_path) - .await?; - let object = self.paths.table_entry_path(table_bucket, &namespace, &table); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; - let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { - return Err(TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - table_bucket, - namespace.public_name(), - table.as_str() - ))); - }; - self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await?; - if let Err(err) = self.delete_table_warehouse_index(&entry).await { - if let Err(restore_err) = self - .write_entry_unlocked(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::IfAbsent) - .await - { - tracing::warn!( - table_bucket = %entry.table_bucket, - namespace = %entry.namespace, - table = %entry.table, - table_id = %entry.table_id, - error = %restore_err, - "failed to restore table entry after warehouse index delete failure" - ); - } - tracing::warn!( - table_bucket = %entry.table_bucket, - namespace = %entry.namespace, - table = %entry.table, - table_id = %entry.table_id, - error = %err, - "failed to delete table warehouse index after dropping table" - ); - return Err(err); - } - Ok(()) - } - - async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { - self.write_view_entry(entry, TableCatalogPutPrecondition::IfAbsent).await - } - - async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - let mut entries = Vec::new(); - for object in self - .backend - .list_objects(self.catalog_bucket(), &self.paths.view_entries_prefix(table_bucket, &namespace)) - .await? - { - if !object.ends_with(VIEW_ENTRY_FILE) { - continue; - } - if let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? { - entries.push(entry); - } - } - entries.sort_by(|left, right| left.view.cmp(&right.view)); - Ok(entries) - } - - async fn list_views_page( - &self, - table_bucket: &str, - namespace: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - self.list_entry_page(&self.paths.view_entries_prefix(table_bucket, &namespace), VIEW_ENTRY_FILE, cursor, limit) - .await - } - - async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - let view = parse_table_for_store(view)?; - self.read_entry::(self.catalog_bucket(), &self.paths.view_entry_path(table_bucket, &namespace, &view)) - .await - .map(|entry| entry.map(|(view, _)| view)) - } - - async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { - let namespace = parse_namespace_for_store(&request.namespace)?; - let view = parse_table_for_store(&request.view)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; - let namespace_path = self.paths.namespace_entry_path(&request.table_bucket, &namespace); - let _namespace_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &namespace_path) - .await?; - let view_path = self.paths.view_entry_path(&request.table_bucket, &namespace, &view); - let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &view_path).await?; - let Some((current, current_etag)) = self - .read_view_with_etag_unlocked(&request.table_bucket, &namespace, &view) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!( - "view {}/{}/{}", - request.table_bucket, request.namespace, request.view - ))); - }; - if current.version_token != request.expected_version_token { - return Err(TableCatalogStoreError::Conflict( - "current view version token does not match expected token".to_string(), - )); - } - if current.metadata_location != request.expected_metadata_location { - return Err(TableCatalogStoreError::Conflict( - "current view metadata location does not match expected location".to_string(), - )); - } - if !is_valid_view_metadata_location(&namespace, &view, &request.new_metadata_location) { - return Err(TableCatalogStoreError::Invalid( - "new metadata location must be inside the view metadata directory".to_string(), - )); - } - let Some(new_metadata_object) = self - .backend - .read_object(&request.table_bucket, &request.new_metadata_location) - .await? - else { - return Err(TableCatalogStoreError::NotFound(format!( - "new view metadata object {}", - request.new_metadata_location - ))); - }; - let next_warehouse_location = - view_metadata_warehouse_location(&request.table_bucket, &request.new_metadata_location, &new_metadata_object)?; - - let mut next = current; - next.metadata_location = request.new_metadata_location; - if let Some(warehouse_location) = next_warehouse_location { - next.warehouse_location = warehouse_location; - } - next.version_token = format!("token-{}", Uuid::new_v4()); - next.generation = next.generation.saturating_add(1); - self.write_entry_unlocked( - self.catalog_bucket(), - &view_path, - &next, - TableCatalogPutPrecondition::IfMatch(current_etag), - ) - .await?; - Ok(ViewCommitResult { view: next }) - } - - async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { - let namespace = parse_namespace_for_store(namespace)?; - let view = parse_table_for_store(view)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; - let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); - let _namespace_guard = self - .backend - .acquire_write_lock(self.catalog_bucket(), &namespace_path) - .await?; - let object = self.paths.view_entry_path(table_bucket, &namespace, &view); - let _view_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; - if self - .read_entry_unlocked::(self.catalog_bucket(), &object) - .await? - .is_none() - { - return Err(TableCatalogStoreError::NotFound(format!( - "view {}/{}/{}", - table_bucket, - namespace.public_name(), - view.as_str() - ))); - } - self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await - } - - async fn get_commit_by_id( - &self, - table_bucket: &str, - table_id: &str, - commit_id: &str, - ) -> TableCatalogStoreResult> { - let object = self.paths.commit_log_entry_path(table_bucket, table_id, commit_id); - self.read_commit_by_path(&object).await - } - - async fn get_commit_by_idempotency_key( - &self, - table_bucket: &str, - table_id: &str, - idempotency_key: &str, - ) -> TableCatalogStoreResult> { - let object = self - .paths - .commit_idempotency_entry_path(table_bucket, table_id, idempotency_key); - self.read_commit_by_path(&object).await - } -} - -#[async_trait::async_trait] -impl TableCatalogStore for ConfiguredTableCatalogStore -where - B: TableCatalogObjectBackend, -{ - async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.get_table_bucket(table_bucket).await, - Self::DurableStrong(store) => store.get_table_bucket(table_bucket).await, - } - } - - async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { - match self { - Self::ObjectBacked(store) => store.put_table_bucket(entry).await, - Self::DurableStrong(store) => store.put_table_bucket(entry).await, - } - } - - async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> { - match self { - Self::ObjectBacked(store) => store.create_namespace(entry).await, - Self::DurableStrong(store) => store.create_namespace(entry).await, - } - } - - async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.list_namespaces(table_bucket).await, - Self::DurableStrong(store) => store.list_namespaces(table_bucket).await, - } - } - - async fn list_namespaces_page( - &self, - table_bucket: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.list_namespaces_page(table_bucket, cursor, limit).await, - Self::DurableStrong(store) => store.list_namespaces_page(table_bucket, cursor, limit).await, - } - } - - async fn get_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.get_namespace(table_bucket, namespace).await, - Self::DurableStrong(store) => store.get_namespace(table_bucket, namespace).await, - } - } - - async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()> { - match self { - Self::ObjectBacked(store) => store.drop_namespace(table_bucket, namespace).await, - Self::DurableStrong(store) => store.drop_namespace(table_bucket, namespace).await, - } - } - - async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { - match self { - Self::ObjectBacked(store) => store.create_table(entry).await, - Self::DurableStrong(store) => store.create_table(entry).await, - } - } - - async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { - match self { - Self::ObjectBacked(store) => store.register_table(entry).await, - Self::DurableStrong(store) => store.register_table(entry).await, - } - } - - async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.list_tables(table_bucket, namespace).await, - Self::DurableStrong(store) => store.list_tables(table_bucket, namespace).await, - } - } - - async fn list_tables_page( - &self, - table_bucket: &str, - namespace: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.list_tables_page(table_bucket, namespace, cursor, limit).await, - Self::DurableStrong(store) => store.list_tables_page(table_bucket, namespace, cursor, limit).await, - } - } - - async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.load_table(table_bucket, namespace, table).await, - Self::DurableStrong(store) => store.load_table(table_bucket, namespace, table).await, - } - } - - async fn resolve_table_data_plane_resource( - &self, - table_bucket: &str, - object: &str, - ) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.resolve_table_data_plane_resource(table_bucket, object).await, - Self::DurableStrong(store) => store.resolve_table_data_plane_resource(table_bucket, object).await, - } - } - - async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => store.commit_table(request).await, - Self::DurableStrong(store) => store.commit_table(request).await, - } - } - - async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { - match self { - Self::ObjectBacked(store) => store.drop_table(table_bucket, namespace, table).await, - Self::DurableStrong(store) => store.drop_table(table_bucket, namespace, table).await, - } - } - - async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { - match self { - Self::ObjectBacked(store) => store.create_view(entry).await, - Self::DurableStrong(store) => store.create_view(entry).await, - } - } - - async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.list_views(table_bucket, namespace).await, - Self::DurableStrong(store) => store.list_views(table_bucket, namespace).await, - } - } - - async fn list_views_page( - &self, - table_bucket: &str, - namespace: &str, - cursor: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.list_views_page(table_bucket, namespace, cursor, limit).await, - Self::DurableStrong(store) => store.list_views_page(table_bucket, namespace, cursor, limit).await, - } - } - - async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.load_view(table_bucket, namespace, view).await, - Self::DurableStrong(store) => store.load_view(table_bucket, namespace, view).await, - } - } - - async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => store.replace_view(request).await, - Self::DurableStrong(store) => store.replace_view(request).await, - } - } - - async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { - match self { - Self::ObjectBacked(store) => store.drop_view(table_bucket, namespace, view).await, - Self::DurableStrong(store) => store.drop_view(table_bucket, namespace, view).await, - } - } - - async fn get_commit_by_id( - &self, - table_bucket: &str, - table_id: &str, - commit_id: &str, - ) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.get_commit_by_id(table_bucket, table_id, commit_id).await, - Self::DurableStrong(store) => store.get_commit_by_id(table_bucket, table_id, commit_id).await, - } - } - - async fn get_commit_by_idempotency_key( - &self, - table_bucket: &str, - table_id: &str, - idempotency_key: &str, - ) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => { - store - .get_commit_by_idempotency_key(table_bucket, table_id, idempotency_key) - .await - } - Self::DurableStrong(store) => { - store - .get_commit_by_idempotency_key(table_bucket, table_id, idempotency_key) - .await - } - } - } -} - -impl ConfiguredTableCatalogStore -where - B: TableCatalogObjectBackend, -{ - pub(crate) async fn get_table_maintenance_config( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => store.get_table_maintenance_config(table_bucket, namespace, table).await, - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance config")), - } - } - - pub(crate) async fn put_table_maintenance_config( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - config: TableMaintenanceConfig, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => { - store - .put_table_maintenance_config(table_bucket, namespace, table, config) - .await - } - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance config")), - } - } - - pub(crate) async fn get_table_metadata_maintenance_report( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - job_id: &str, - ) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => { - store - .get_table_metadata_maintenance_report(table_bucket, namespace, table, job_id) - .await - } - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance report")), - } - } - - pub(crate) async fn get_table_maintenance_scheduler_report( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => { - store - .get_table_maintenance_scheduler_report(table_bucket, namespace, table) - .await - } - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance scheduler")), - } - } - - pub(crate) async fn run_table_maintenance_scheduler_once( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - scheduler_id: String, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => { - store - .run_table_maintenance_scheduler_once(table_bucket, namespace, table, scheduler_id) - .await - } - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance scheduler")), - } - } - - pub(crate) async fn apply_table_maintenance_quarantine_operation( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - job_id: &str, - request: TableMaintenanceQuarantineOperationRequest, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => { - store - .apply_table_maintenance_quarantine_operation(table_bucket, namespace, table, job_id, request) - .await - } - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance quarantine")), - } - } - - pub(crate) async fn run_table_metadata_maintenance_worker_once( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - worker_id: String, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => { - store - .run_table_metadata_maintenance_worker_once(table_bucket, namespace, table, worker_id) - .await - } - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance worker")), - } - } - - pub(crate) async fn heartbeat_table_metadata_maintenance_job( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - job_id: &str, - lease_id: &str, - worker_id: &str, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => { - store - .heartbeat_table_metadata_maintenance_job(table_bucket, namespace, table, job_id, lease_id, worker_id) - .await - } - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance heartbeat")), - } - } - - pub(crate) async fn export_table_catalog_entry( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => store.export_table_catalog_entry(table_bucket, namespace, table).await, - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("catalog export")), - } - } - - pub(crate) async fn diagnose_table_catalog( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - retain_recent_metadata_files: usize, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => { - store - .diagnose_table_catalog(table_bucket, namespace, table, retain_recent_metadata_files) - .await - } - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("catalog diagnostics")), - } - } - - pub(crate) async fn recover_table_commits( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => store.recover_table_commits(table_bucket, namespace, table).await, - Self::DurableStrong(store) => store.plan_table_commit_recovery(table_bucket, namespace, table).await, - } - } - - pub(crate) async fn get_external_catalog_bridge( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> TableCatalogStoreResult> { - match self { - Self::ObjectBacked(store) => store.get_external_catalog_bridge(table_bucket, namespace, table).await, - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("external catalog bridge")), - } - } - - pub(crate) async fn put_external_catalog_bridge( - &self, - entry: ExternalCatalogBridgeEntry, - ) -> TableCatalogStoreResult { - match self { - Self::ObjectBacked(store) => store.put_external_catalog_bridge(entry).await, - Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("external catalog bridge")), - } - } -} - -pub(crate) struct EcStoreTableCatalogObjectBackend { - store: Arc, -} - -impl Clone for EcStoreTableCatalogObjectBackend { - fn clone(&self) -> Self { - Self { - store: self.store.clone(), - } - } -} - -impl EcStoreTableCatalogObjectBackend -where - S: TableCatalogStorage, -{ - pub fn new(store: Arc) -> Self { - Self { store } - } -} - -pub(crate) type EcStoreTableCatalogStore = ConfiguredTableCatalogStore>; - -#[async_trait::async_trait] -impl TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend -where - S: TableCatalogStorage, -{ - async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - self.read_object_with_options(bucket, object, ObjectOptions::default()).await - } - - async fn read_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - self.read_object_with_options( - bucket, - object, - ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - } - - async fn object_metadata(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - match self.store.get_object_info(bucket, object, &ObjectOptions::default()).await { - Ok(info) => Ok(Some(TableCatalogObjectMetadata { - etag: info.etag, - mod_time: info.mod_time, - })), - Err(err) if is_missing_storage_error(&err) => Ok(None), - Err(err) => Err(storage_error_to_catalog("stat catalog object", err)), - } - } - - async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { - match self.store.get_object_info(bucket, object, &ObjectOptions::default()).await { - Ok(_) => Ok(true), - Err(err) if is_missing_storage_error(&err) => Ok(false), - Err(err) => Err(storage_error_to_catalog("check catalog object", err)), - } - } - - async fn put_object( - &self, - bucket: &str, - object: &str, - data: Vec, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> { - self.put_object_with_options(bucket, object, data, precondition, false).await - } - - async fn put_object_unlocked( - &self, - bucket: &str, - object: &str, - data: Vec, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> { - self.put_object_with_options(bucket, object, data, precondition, true).await - } - - async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { - match self.store.delete_object(bucket, object, ObjectOptions::default()).await { - Ok(_) => Ok(()), - Err(err) if is_missing_storage_error(&err) => Ok(()), - Err(err) => Err(storage_error_to_catalog("delete catalog object", err)), - } - } - - async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { - match self - .store - .delete_object( - bucket, - object, - ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - { - Ok(_) => Ok(()), - Err(err) if is_missing_storage_error(&err) => Ok(()), - Err(err) => Err(storage_error_to_catalog("delete catalog object", err)), - } - } - - async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult> { - let mut continuation = None; - let mut objects = BTreeSet::new(); - let max_keys = i32::try_from(TABLE_CATALOG_LIST_MAX_KEYS) - .map_err(|_| TableCatalogStoreError::Internal("catalog list limit exceeds storage API range".to_string()))?; - - loop { - let result = self - .store - .clone() - .list_objects_v2(bucket, prefix, continuation, None, max_keys, false, None, false) - .await - .map_err(|err| storage_error_to_catalog("list catalog objects", err))?; - - for object in result.objects { - objects.insert(object.name); - } - - if !result.is_truncated { - break; - } - - let Some(next) = result.next_continuation_token else { - break; - }; - continuation = Some(next); - } - - Ok(objects.into_iter().collect()) - } - - async fn list_objects_page( - &self, - bucket: &str, - prefix: &str, - start_after: Option<&str>, - limit: NonZeroUsize, - ) -> TableCatalogStoreResult { - let max_keys = i32::try_from(limit.get()) - .map_err(|_| TableCatalogStoreError::Invalid("catalog page size exceeds storage API range".to_string()))?; - let result = self - .store - .clone() - .list_objects_v2(bucket, prefix, None, None, max_keys, false, start_after.map(str::to_string), false) - .await - .map_err(|err| storage_error_to_catalog("list catalog object page", err))?; - let is_truncated = result.is_truncated; - let objects = result.objects.into_iter().map(|object| object.name).collect::>(); - Ok(TableCatalogObjectListPage { - objects: objects.into_iter().collect(), - is_truncated, - }) - } - - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - let lock = self - .store - .new_ns_lock(bucket, object) - .await - .map_err(|err| storage_error_to_catalog("create catalog table lock", err))?; - let guard = lock - .get_write_lock(get_lock_acquire_timeout()) - .await - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog table lock: {err}")))?; - Ok(Box::new(guard)) - } - - async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - let lock = self - .store - .new_ns_lock(bucket, object) - .await - .map_err(|err| storage_error_to_catalog("create catalog migration lock", err))?; - let guard = lock - .get_read_lock(get_lock_acquire_timeout()) - .await - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog migration lock: {err}")))?; - Ok(Box::new(guard)) - } -} - -impl EcStoreTableCatalogObjectBackend -where - S: TableCatalogStorage, -{ - async fn read_object_with_options( - &self, - bucket: &str, - object: &str, - opts: ObjectOptions, - ) -> TableCatalogStoreResult> { - let info = match self.store.get_object_info(bucket, object, &opts).await { - Ok(info) => info, - Err(err) if is_missing_storage_error(&err) => return Ok(None), - Err(err) => return Err(storage_error_to_catalog("read catalog object info", err)), - }; - let mut reader = match self - .store - .get_object_reader(bucket, object, None, HeaderMap::new(), &opts) - .await - { - Ok(reader) => reader, - Err(err) if is_missing_storage_error(&err) => return Ok(None), - Err(err) => return Err(storage_error_to_catalog("read catalog object", err)), - }; - let mut data = Vec::new(); - reader - .stream - .read_to_end(&mut data) - .await - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to read catalog object {bucket}/{object}: {err}")))?; - Ok(Some(TableCatalogObject { - data, - etag: info.etag, - mod_time: info.mod_time, - })) - } - - async fn put_object_with_options( - &self, - bucket: &str, - object: &str, - data: Vec, - precondition: TableCatalogPutPrecondition, - no_lock: bool, - ) -> TableCatalogStoreResult<()> { - let mut reader = PutObjReader::from_vec(data); - let opts = ObjectOptions { - http_preconditions: http_preconditions_for_catalog_put(precondition), - no_lock, - ..Default::default() - }; - self.store - .put_object(bucket, object, &mut reader, &opts) - .await - .map(|_| ()) - .map_err(|err| storage_error_to_catalog("write catalog object", err)) - } -} - -fn parse_namespace_for_store(namespace: &str) -> TableCatalogStoreResult { - Namespace::parse(namespace).map_err(|err| TableCatalogStoreError::Invalid(format!("invalid namespace: {err}"))) -} - -fn parse_table_for_store(table: &str) -> TableCatalogStoreResult { - IdentifierSegment::parse(table).map_err(|err| TableCatalogStoreError::Invalid(format!("invalid table name: {err}"))) -} - -fn insert_metadata_maintenance_reason( - reasons_by_location: &mut BTreeMap>, - metadata_location: String, - reason: TableMetadataMaintenanceReason, -) { - reasons_by_location.entry(metadata_location).or_default().insert(reason); -} - -fn metadata_maintenance_object_reports( - reasons_by_location: BTreeMap>, -) -> Vec { - reasons_by_location - .into_iter() - .map(|(metadata_location, reasons)| { - let reasons = reasons.into_iter().collect::>(); - let state = if reasons.contains(&TableMetadataMaintenanceReason::SafetyWindowSatisfied) { - TableMetadataMaintenanceObjectState::Deletable - } else if reasons.contains(&TableMetadataMaintenanceReason::SafetyWindowPending) { - TableMetadataMaintenanceObjectState::PendingSafetyWindow - } else { - TableMetadataMaintenanceObjectState::Retained - }; - TableMetadataMaintenanceObjectReport { - metadata_location, - state, - reasons, - } - }) - .collect() -} - -#[derive(Debug, Clone)] -struct TableMetadataMaintenanceReferencedObjectAccumulator { - object_kind: TableMetadataMaintenanceObjectKind, - state: TableMetadataMaintenanceObjectState, - reasons: BTreeSet, -} - -fn insert_referenced_object_report( - reports: &mut BTreeMap, - object_location: String, - object_kind: TableMetadataMaintenanceObjectKind, - state: TableMetadataMaintenanceObjectState, - reason: TableMetadataMaintenanceReason, -) { - let report = reports - .entry(object_location) - .or_insert_with(|| TableMetadataMaintenanceReferencedObjectAccumulator { - object_kind, - state: TableMetadataMaintenanceObjectState::Retained, - reasons: BTreeSet::new(), - }); - if state == TableMetadataMaintenanceObjectState::ManualReviewRequired { - report.state = TableMetadataMaintenanceObjectState::ManualReviewRequired; - } - report.reasons.insert(reason); -} - -async fn metadata_maintenance_referenced_object_reports( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - current_metadata: &serde_json::Value, - retained_metadata_locations: &[String], -) -> TableCatalogStoreResult> -where - B: TableCatalogObjectBackend, -{ - let mut reports = BTreeMap::::new(); - metadata_maintenance_referenced_object_reports_for_metadata( - backend, - table_bucket, - namespace, - table, - warehouse_object_prefix, - current_metadata, - &mut reports, - ) - .await?; - - for metadata_location in retained_metadata_locations { - let Some(metadata_object) = backend.read_object(table_bucket, metadata_location).await? else { - insert_referenced_object_report( - &mut reports, - metadata_location.clone(), - TableMetadataMaintenanceObjectKind::MetadataFile, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnreadableMetadata, - ); - continue; - }; - let Ok(metadata) = serde_json::from_slice::(&metadata_object.data) else { - insert_referenced_object_report( - &mut reports, - metadata_location.clone(), - TableMetadataMaintenanceObjectKind::MetadataFile, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnreadableMetadata, - ); - continue; - }; - if !metadata.is_object() { - insert_referenced_object_report( - &mut reports, - metadata_location.clone(), - TableMetadataMaintenanceObjectKind::MetadataFile, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnreadableMetadata, - ); - continue; - } - metadata_maintenance_referenced_object_reports_for_metadata( - backend, - table_bucket, - namespace, - table, - warehouse_object_prefix, - &metadata, - &mut reports, - ) - .await?; - } - - Ok(reports - .into_iter() - .map(|(object_location, report)| TableMetadataMaintenanceReferencedObjectReport { - object_location, - object_kind: report.object_kind, - state: report.state, - reasons: report.reasons.into_iter().collect(), - }) - .collect()) -} - -async fn metadata_maintenance_referenced_object_reports_for_metadata( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - metadata: &serde_json::Value, - reports: &mut BTreeMap, -) -> TableCatalogStoreResult<()> -where - B: TableCatalogObjectBackend, -{ - let Some(snapshots) = metadata.get("snapshots").and_then(serde_json::Value::as_array) else { - return Ok(()); - }; - - for snapshot in snapshots { - if let Some(manifest_list_location) = snapshot.get("manifest-list").and_then(serde_json::Value::as_str) { - metadata_maintenance_referenced_manifest_list( - backend, - table_bucket, - namespace, - table, - warehouse_object_prefix, - manifest_list_location, - reports, - ) - .await?; - continue; - } - - let Some(manifests) = snapshot.get("manifests").and_then(serde_json::Value::as_array) else { - continue; - }; - for manifest in manifests { - let Some(manifest_location) = manifest.as_str() else { - insert_referenced_object_report( - reports, - "snapshots[].manifests".to_string(), - TableMetadataMaintenanceObjectKind::ManifestFile, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - continue; - }; - metadata_maintenance_referenced_manifest_file( - backend, - table_bucket, - namespace, - table, - warehouse_object_prefix, - manifest_location, - reports, - ) - .await?; - } - } - - Ok(()) -} - -async fn metadata_maintenance_referenced_manifest_list( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - manifest_list_location: &str, - reports: &mut BTreeMap, -) -> TableCatalogStoreResult<()> -where - B: TableCatalogObjectBackend, -{ - let Some(manifest_list_key) = table_catalog_object_key_from_location(table_bucket, manifest_list_location) else { - insert_referenced_object_report( - reports, - manifest_list_location.to_string(), - TableMetadataMaintenanceObjectKind::ManifestList, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - return Ok(()); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &manifest_list_key) - != Some(TableMetadataMaintenanceObjectKind::ManifestList) - { - insert_referenced_object_report( - reports, - manifest_list_key, - TableMetadataMaintenanceObjectKind::ManifestList, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - return Ok(()); - } - insert_referenced_object_report( - reports, - manifest_list_key.clone(), - TableMetadataMaintenanceObjectKind::ManifestList, - TableMetadataMaintenanceObjectState::Retained, - TableMetadataMaintenanceReason::ManifestList, - ); - - let Some(manifest_list_object) = backend.read_object(table_bucket, &manifest_list_key).await? else { - mark_referenced_object_manual_review( - reports, - &manifest_list_key, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - return Ok(()); - }; - let Ok(manifest_paths) = manifest_paths_from_manifest_list_avro(&manifest_list_object.data) else { - mark_referenced_object_manual_review( - reports, - &manifest_list_key, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - return Ok(()); - }; - for manifest_location in manifest_paths { - metadata_maintenance_referenced_manifest_file( - backend, - table_bucket, - namespace, - table, - warehouse_object_prefix, - &manifest_location, - reports, - ) - .await?; - } - - Ok(()) -} - -async fn metadata_maintenance_referenced_manifest_file( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - manifest_location: &str, - reports: &mut BTreeMap, -) -> TableCatalogStoreResult<()> -where - B: TableCatalogObjectBackend, -{ - let Some(manifest_key) = table_catalog_object_key_from_location(table_bucket, manifest_location) else { - insert_referenced_object_report( - reports, - manifest_location.to_string(), - TableMetadataMaintenanceObjectKind::ManifestFile, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - return Ok(()); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &manifest_key) - != Some(TableMetadataMaintenanceObjectKind::ManifestFile) - { - insert_referenced_object_report( - reports, - manifest_key, - TableMetadataMaintenanceObjectKind::ManifestFile, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - return Ok(()); - } - insert_referenced_object_report( - reports, - manifest_key.clone(), - TableMetadataMaintenanceObjectKind::ManifestFile, - TableMetadataMaintenanceObjectState::Retained, - TableMetadataMaintenanceReason::ManifestFile, - ); - - let Some(manifest_object) = backend.read_object(table_bucket, &manifest_key).await? else { - mark_referenced_object_manual_review(reports, &manifest_key, TableMetadataMaintenanceReason::UnsupportedManifestAvro); - return Ok(()); - }; - let Ok(file_references) = file_references_from_manifest_avro(&manifest_object.data) else { - mark_referenced_object_manual_review(reports, &manifest_key, TableMetadataMaintenanceReason::UnsupportedManifestAvro); - return Ok(()); - }; - for (file_location, object_kind) in file_references { - let Some(file_key) = table_catalog_object_key_from_location(table_bucket, &file_location) else { - insert_referenced_object_report( - reports, - file_location, - object_kind, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - continue; - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &file_key) != Some(object_kind.clone()) { - insert_referenced_object_report( - reports, - file_key, - object_kind, - TableMetadataMaintenanceObjectState::ManualReviewRequired, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ); - continue; - } - insert_referenced_object_report( - reports, - file_key, - object_kind.clone(), - TableMetadataMaintenanceObjectState::Retained, - table_metadata_maintenance_reason_for_object_kind(&object_kind), - ); - } - - Ok(()) -} - -fn mark_referenced_object_manual_review( - reports: &mut BTreeMap, - object_location: &str, - reason: TableMetadataMaintenanceReason, -) { - if let Some(report) = reports.get_mut(object_location) { - report.state = TableMetadataMaintenanceObjectState::ManualReviewRequired; - report.reasons.insert(reason); - } -} - -fn manifest_paths_from_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult> { - Ok(manifest_list_references_from_manifest_list_avro(data)? - .into_iter() - .map(|reference| reference.manifest_path) - .collect()) -} - -pub(crate) fn manifest_list_references_from_manifest_list_avro( - data: &[u8], -) -> TableCatalogStoreResult> { - let reader = apache_avro::Reader::new(data) - .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest list Avro: {err}")))?; - let mut manifest_paths = Vec::new(); - for value in reader { - let value = - value.map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest list record: {err}")))?; - let manifest_path = avro_record_field(&value, "manifest_path") - .and_then(avro_string_value) - .ok_or_else(|| TableCatalogStoreError::Invalid("manifest list entry missing manifest_path".to_string()))?; - manifest_paths.push(ManifestListReference { - manifest_path: manifest_path.to_string(), - partition_spec_id: avro_record_field(&value, "partition_spec_id").and_then(avro_i32_value), - sequence_number: avro_record_field(&value, "sequence_number").and_then(avro_i64_value), - added_snapshot_id: avro_record_field(&value, "added_snapshot_id").and_then(avro_i64_value), - }); - } - Ok(manifest_paths) -} - -fn file_references_from_manifest_avro(data: &[u8]) -> TableCatalogStoreResult> { - Ok(data_file_references_from_manifest_avro(data)? - .into_iter() - .map(|reference| (reference.location, reference.object_kind)) - .collect()) -} - -pub(crate) fn data_file_references_from_manifest_avro(data: &[u8]) -> TableCatalogStoreResult> { - let reader = apache_avro::Reader::new(data) - .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest Avro: {err}")))?; - let mut files = Vec::new(); - for value in reader { - let value = value.map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest record: {err}")))?; - let data_file = avro_record_field(&value, "data_file") - .ok_or_else(|| TableCatalogStoreError::Invalid("manifest entry missing data_file".to_string()))?; - let file_path = avro_record_field(data_file, "file_path") - .and_then(avro_string_value) - .ok_or_else(|| TableCatalogStoreError::Invalid("manifest data file missing file_path".to_string()))?; - let content = avro_record_field(data_file, "content") - .and_then(avro_i32_value) - .ok_or_else(|| TableCatalogStoreError::Invalid("manifest data file missing content".to_string()))?; - let (content, object_kind) = match content { - 0 => (ManifestDataFileContent::Data, TableMetadataMaintenanceObjectKind::DataFile), - 1 => (ManifestDataFileContent::PositionDelete, TableMetadataMaintenanceObjectKind::DeleteFile), - 2 => (ManifestDataFileContent::EqualityDelete, TableMetadataMaintenanceObjectKind::DeleteFile), - _ => continue, - }; - files.push(ManifestDataFileReference { - location: file_path.to_string(), - content, - object_kind, - entry_status: avro_record_field(&value, "status").and_then(avro_i32_value), - snapshot_id: avro_record_field(&value, "snapshot_id").and_then(avro_i64_value), - sequence_number: avro_record_field(&value, "sequence_number").and_then(avro_i64_value), - file_sequence_number: avro_record_field(&value, "file_sequence_number").and_then(avro_i64_value), - record_count: avro_record_field(data_file, "record_count") - .and_then(avro_i64_value) - .and_then(|value| u64::try_from(value).ok()), - file_size_bytes: avro_record_field(data_file, "file_size_in_bytes") - .and_then(avro_i64_value) - .and_then(|value| u64::try_from(value).ok()), - partition: avro_record_field(data_file, "partition") - .and_then(avro_record_value_fields) - .unwrap_or_default(), - sort_order_id: avro_record_field(data_file, "sort_order_id").and_then(avro_i32_value), - }); - } - Ok(files) -} - -fn avro_record_field<'a>(value: &'a apache_avro::types::Value, name: &str) -> Option<&'a apache_avro::types::Value> { - let value = avro_non_union_value(value); - let apache_avro::types::Value::Record(fields) = value else { - return None; - }; - fields - .iter() - .find_map(|(field_name, field_value)| (field_name == name).then_some(avro_non_union_value(field_value))) -} - -fn avro_record_value_fields(value: &apache_avro::types::Value) -> Option> { - let value = avro_non_union_value(value); - let apache_avro::types::Value::Record(fields) = value else { - return None; - }; - Some( - fields - .iter() - .map(|(field_name, field_value)| (field_name.clone(), avro_non_union_value(field_value).clone())) - .collect(), - ) -} - -fn avro_non_union_value(value: &apache_avro::types::Value) -> &apache_avro::types::Value { - match value { - apache_avro::types::Value::Union(_, inner) => avro_non_union_value(inner), - value => value, - } -} - -fn avro_string_value(value: &apache_avro::types::Value) -> Option<&str> { - match avro_non_union_value(value) { - apache_avro::types::Value::String(value) => Some(value), - _ => None, - } -} - -fn avro_i32_value(value: &apache_avro::types::Value) -> Option { - match avro_non_union_value(value) { - apache_avro::types::Value::Int(value) => Some(*value), - _ => None, - } -} - -fn avro_i64_value(value: &apache_avro::types::Value) -> Option { - match avro_non_union_value(value) { - apache_avro::types::Value::Long(value) => Some(*value), - _ => None, - } -} - -pub(crate) fn table_catalog_object_key_from_location(table_bucket: &str, location: &str) -> Option { - let object = if let Some(location) = location.strip_prefix("s3://") { - let (bucket, object) = location.split_once('/')?; - if bucket != table_bucket { - return None; - } - object - } else { - location - }; - - if object.is_empty() - || object.starts_with('/') - || object.contains("..") - || object.contains('\\') - || object.bytes().any(|byte| byte.is_ascii_control()) - { - return None; - } - - Some(object.to_string()) -} - -pub(crate) fn table_maintenance_object_kind( - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - object_location: &str, -) -> Option { - let metadata_prefix = format!("{}/", default_table_metadata_dir_path(namespace, table)); - if let Some(kind) = table_maintenance_metadata_object_kind(&metadata_prefix, object_location) { - return Some(kind); - } - - let data_prefix = format!("{}/", default_table_data_dir_path(namespace, table)); - if object_location - .strip_prefix(&data_prefix) - .is_some_and(is_valid_table_maintenance_nested_object) - { - return Some(TableMetadataMaintenanceObjectKind::DataFile); - } - - let delete_prefix = format!("{}/", default_table_delete_dir_path(namespace, table)); - if object_location - .strip_prefix(&delete_prefix) - .is_some_and(is_valid_table_maintenance_nested_object) - { - return Some(TableMetadataMaintenanceObjectKind::DeleteFile); - } - - if let Some(warehouse_object_prefix) = warehouse_object_prefix { - let metadata_prefix = format!("{warehouse_object_prefix}{METADATA_DIR}/"); - if let Some(kind) = table_maintenance_metadata_object_kind(&metadata_prefix, object_location) { - return Some(kind); - } - - let data_prefix = format!("{warehouse_object_prefix}{DATA_DIR}/"); - if object_location - .strip_prefix(&data_prefix) - .is_some_and(is_valid_table_maintenance_nested_object) - { - return Some(TableMetadataMaintenanceObjectKind::DataFile); - } - - let delete_prefix = format!("{warehouse_object_prefix}{DELETE_DIR}/"); - if object_location - .strip_prefix(&delete_prefix) - .is_some_and(is_valid_table_maintenance_nested_object) - { - return Some(TableMetadataMaintenanceObjectKind::DeleteFile); - } - } - - None -} - -fn table_maintenance_metadata_object_kind( - metadata_prefix: &str, - object_location: &str, -) -> Option { - let file_name = object_location.strip_prefix(metadata_prefix)?; - if file_name.is_empty() - || file_name.contains('/') - || file_name.contains('\\') - || file_name.contains("..") - || file_name.bytes().any(|byte| byte.is_ascii_control()) - || !file_name.ends_with(".avro") - { - return None; - } - if file_name.starts_with("snap-") { - return Some(TableMetadataMaintenanceObjectKind::ManifestList); - } - Some(TableMetadataMaintenanceObjectKind::ManifestFile) -} - -fn is_valid_table_maintenance_nested_object(suffix: &str) -> bool { - !suffix.is_empty() - && !suffix.starts_with('/') - && !suffix.contains("..") - && !suffix.contains('\\') - && !suffix.bytes().any(|byte| byte.is_ascii_control()) -} - -fn table_metadata_maintenance_reason_for_object_kind( - object_kind: &TableMetadataMaintenanceObjectKind, -) -> TableMetadataMaintenanceReason { - match object_kind { - TableMetadataMaintenanceObjectKind::MetadataFile => TableMetadataMaintenanceReason::CurrentMetadata, - TableMetadataMaintenanceObjectKind::ManifestList => TableMetadataMaintenanceReason::ManifestList, - TableMetadataMaintenanceObjectKind::ManifestFile => TableMetadataMaintenanceReason::ManifestFile, - TableMetadataMaintenanceObjectKind::DataFile => TableMetadataMaintenanceReason::DataFile, - TableMetadataMaintenanceObjectKind::DeleteFile => TableMetadataMaintenanceReason::DeleteFile, - } -} - -fn metadata_maintenance_reachability_graph_report( - metadata_file_count: usize, - referenced_object_reports: &[TableMetadataMaintenanceReferencedObjectReport], -) -> TableMaintenanceReachabilityGraphReport { - let manifest_list_count = referenced_object_reports - .iter() - .filter(|report| report.object_kind == TableMetadataMaintenanceObjectKind::ManifestList) - .count(); - let manifest_file_count = referenced_object_reports - .iter() - .filter(|report| report.object_kind == TableMetadataMaintenanceObjectKind::ManifestFile) - .count(); - let data_file_count = referenced_object_reports - .iter() - .filter(|report| report.object_kind == TableMetadataMaintenanceObjectKind::DataFile) - .count(); - let delete_file_count = referenced_object_reports - .iter() - .filter(|report| report.object_kind == TableMetadataMaintenanceObjectKind::DeleteFile) - .count(); - let manual_review_count = referenced_object_reports - .iter() - .filter(|report| report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired) - .count(); - let mut reasons = BTreeSet::from([TableMaintenanceReachabilityGraphReason::MetadataJsonParsed]); - if manifest_list_count > 0 { - reasons.insert(TableMaintenanceReachabilityGraphReason::ManifestListAvroReferenced); - } - if referenced_object_reports.iter().any(|report| { - report - .reasons - .contains(&TableMetadataMaintenanceReason::UnsupportedManifestAvro) - }) { - reasons.insert(TableMaintenanceReachabilityGraphReason::ManifestAvroReaderUnavailable); - } - - TableMaintenanceReachabilityGraphReport { - status: if manual_review_count == 0 { - TableMaintenanceReachabilityGraphStatus::Complete - } else { - TableMaintenanceReachabilityGraphStatus::ManualReviewRequired - }, - metadata_file_count, - manifest_list_count, - manifest_file_count, - data_file_count, - delete_file_count, - manual_review_count, - reasons: reasons.into_iter().collect(), - } -} - -async fn metadata_maintenance_object_cleanup_reports( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - referenced_object_reports: &[TableMetadataMaintenanceReferencedObjectReport], - now: OffsetDateTime, -) -> TableCatalogStoreResult<(usize, Vec, Vec, Vec)> -where - B: TableCatalogObjectBackend, -{ - let scanned_objects = - table_maintenance_cleanup_objects(backend, table_bucket, namespace, table, warehouse_object_prefix).await?; - if referenced_object_reports - .iter() - .any(|report| report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired) - { - return Ok((scanned_objects.len(), Vec::new(), Vec::new(), Vec::new())); - } - - let referenced_locations = referenced_object_reports - .iter() - .filter_map(|report| table_catalog_object_key_from_location(table_bucket, &report.object_location)) - .collect::>(); - let mut cleanup_candidate_locations = Vec::new(); - let mut deletable_object_locations = Vec::new(); - let mut cleanup_reports = Vec::new(); - - for (object_location, object_kind) in scanned_objects { - if referenced_locations.contains(&object_location) { - continue; - } - let mut reasons = BTreeSet::from([ - table_metadata_maintenance_reason_for_object_kind(&object_kind), - TableMetadataMaintenanceReason::NoCurrentReachability, - ]); - let state = match backend.read_object(table_bucket, &object_location).await? { - Some(object) if metadata_candidate_is_past_safety_window(object.mod_time, now) => { - reasons.insert(TableMetadataMaintenanceReason::SafetyWindowSatisfied); - cleanup_candidate_locations.push(object_location.clone()); - deletable_object_locations.push(object_location.clone()); - TableMetadataMaintenanceObjectState::Deletable - } - _ => { - reasons.insert(TableMetadataMaintenanceReason::SafetyWindowPending); - cleanup_candidate_locations.push(object_location.clone()); - TableMetadataMaintenanceObjectState::PendingSafetyWindow - } - }; - cleanup_reports.push(TableMetadataMaintenanceObjectCleanupReport { - object_location, - object_kind, - state, - reasons: reasons.into_iter().collect(), - }); - } - - Ok(( - referenced_locations.len() + cleanup_reports.len(), - cleanup_candidate_locations, - deletable_object_locations, - cleanup_reports, - )) -} - -async fn table_maintenance_cleanup_objects( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, -) -> TableCatalogStoreResult> -where - B: TableCatalogObjectBackend, -{ - let mut objects = BTreeMap::new(); - let mut metadata_prefixes = vec![format!("{}/", default_table_metadata_dir_path(namespace, table))]; - let mut data_prefixes = vec![format!("{}/", default_table_data_dir_path(namespace, table))]; - let mut delete_prefixes = vec![format!("{}/", default_table_delete_dir_path(namespace, table))]; - if let Some(warehouse_object_prefix) = warehouse_object_prefix { - metadata_prefixes.push(format!("{warehouse_object_prefix}{METADATA_DIR}/")); - data_prefixes.push(format!("{warehouse_object_prefix}{DATA_DIR}/")); - delete_prefixes.push(format!("{warehouse_object_prefix}{DELETE_DIR}/")); - } - metadata_prefixes.sort(); - metadata_prefixes.dedup(); - data_prefixes.sort(); - data_prefixes.dedup(); - delete_prefixes.sort(); - delete_prefixes.dedup(); - - for metadata_prefix in metadata_prefixes { - for object in backend.list_objects(table_bucket, &metadata_prefix).await? { - if let Some(kind) = table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &object) - && matches!( - kind, - TableMetadataMaintenanceObjectKind::ManifestList | TableMetadataMaintenanceObjectKind::ManifestFile - ) - { - objects.insert(object, kind); - } - } - } - - for data_prefix in data_prefixes { - for object in backend.list_objects(table_bucket, &data_prefix).await? { - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &object) - == Some(TableMetadataMaintenanceObjectKind::DataFile) - { - objects.insert(object, TableMetadataMaintenanceObjectKind::DataFile); - } - } - } - - for delete_prefix in delete_prefixes { - for object in backend.list_objects(table_bucket, &delete_prefix).await? { - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &object) - == Some(TableMetadataMaintenanceObjectKind::DeleteFile) - { - objects.insert(object, TableMetadataMaintenanceObjectKind::DeleteFile); - } - } - } - - Ok(objects) -} - -fn mark_deleted_metadata_object_reports( - object_reports: &mut [TableMetadataMaintenanceObjectReport], - deleted_locations: &BTreeSet, -) { - for object_report in object_reports { - if !deleted_locations.contains(&object_report.metadata_location) { - continue; - } - object_report.state = TableMetadataMaintenanceObjectState::Deleted; - if !object_report - .reasons - .contains(&TableMetadataMaintenanceReason::DeletedByMaintenance) - { - object_report - .reasons - .push(TableMetadataMaintenanceReason::DeletedByMaintenance); - } - } -} - -fn mark_deleted_object_cleanup_reports( - object_reports: &mut [TableMetadataMaintenanceObjectCleanupReport], - deleted_locations: &BTreeSet, -) { - for object_report in object_reports { - if !deleted_locations.contains(&object_report.object_location) { - continue; - } - object_report.state = TableMetadataMaintenanceObjectState::Deleted; - if !object_report - .reasons - .contains(&TableMetadataMaintenanceReason::DeletedByMaintenance) - { - object_report - .reasons - .push(TableMetadataMaintenanceReason::DeletedByMaintenance); - } - } -} - -fn metadata_log_locations( - current_metadata: &serde_json::Value, - namespace: &Namespace, - table: &IdentifierSegment, -) -> BTreeSet { - let mut locations = BTreeSet::new(); - let Some(metadata_log) = current_metadata.get("metadata-log").and_then(serde_json::Value::as_array) else { - return locations; - }; - - for entry in metadata_log { - let Some(metadata_location) = entry.get("metadata-file").and_then(serde_json::Value::as_str) else { - continue; - }; - if is_valid_table_metadata_location(namespace, table, metadata_location) { - locations.insert(metadata_location.to_string()); - } - } - - locations -} - -async fn metadata_locations_for_protected_snapshot_refs( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - current_metadata: &serde_json::Value, - metadata_locations: &[String], -) -> TableCatalogStoreResult> -where - B: TableCatalogObjectBackend, -{ - let protected_snapshot_ids = protected_ref_snapshot_ids(current_metadata); - if protected_snapshot_ids.is_empty() { - return Ok(BTreeSet::new()); - } - - let mut retained = BTreeSet::new(); - for metadata_location in metadata_locations { - if !is_valid_table_metadata_location(namespace, table, metadata_location) { - continue; - } - let Some(metadata_object) = backend.read_object(table_bucket, metadata_location).await? else { - continue; - }; - let Ok(metadata) = serde_json::from_slice::(&metadata_object.data) else { - continue; - }; - if metadata_contains_protected_snapshot_ref(&metadata, &protected_snapshot_ids) { - retained.insert(metadata_location.clone()); - } - } - Ok(retained) -} - -fn protected_ref_snapshot_ids(current_metadata: &serde_json::Value) -> BTreeSet { - let mut snapshot_ids = BTreeSet::new(); - let current_snapshot_id = current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64); - let Some(refs) = current_metadata.get("refs").and_then(serde_json::Value::as_object) else { - return snapshot_ids; - }; - - for reference in refs.values() { - if let Some(snapshot_id) = reference.get("snapshot-id").and_then(serde_json::Value::as_i64) - && Some(snapshot_id) != current_snapshot_id - { - snapshot_ids.insert(snapshot_id); - } - } - snapshot_ids -} - -fn metadata_contains_protected_snapshot_ref(metadata: &serde_json::Value, protected_snapshot_ids: &BTreeSet) -> bool { - let current_snapshot_matches = metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_some_and(|snapshot_id| protected_snapshot_ids.contains(&snapshot_id)); - if current_snapshot_matches { - return true; - } - - let Some(refs) = metadata.get("refs").and_then(serde_json::Value::as_object) else { - return false; - }; - refs.values().any(|reference| { - reference - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_some_and(|snapshot_id| protected_snapshot_ids.contains(&snapshot_id)) - }) -} - -fn metadata_candidate_is_past_safety_window(mod_time: Option, now: OffsetDateTime) -> bool { - let Some(mod_time) = mod_time else { - return false; - }; - mod_time <= now - Duration::seconds(TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS) -} - -#[derive(Debug)] -struct TableSnapshotExpirationDraft { - snapshot_id: Option, - sequence_number: Option, - timestamp_ms: Option, - manifest_list: Option, - reasons: BTreeSet, -} - -fn table_snapshot_expiration_report( - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - entry: &TableEntry, - current_metadata: &serde_json::Value, - config: TableSnapshotExpirationConfig, - now: OffsetDateTime, -) -> TableSnapshotExpirationReport { - let current_snapshot_id = current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64); - let expiration_watermark_ms = unix_timestamp_millis(now).saturating_sub(config.max_snapshot_age_ms); - let (protected_ref_snapshot_ids, user_defined_ref_snapshot_ids, ref_retention_conflict_snapshot_ids) = - snapshot_expiration_ref_state(current_metadata, current_snapshot_id); - let table_retention_property_conflict = snapshot_expiration_table_property_conflicts(current_metadata, &config); - - let mut drafts = snapshot_expiration_drafts(current_metadata, current_snapshot_id); - mark_recent_snapshots_to_keep(&mut drafts, config.min_snapshots_to_keep); - - let mut snapshot_reports = Vec::with_capacity(drafts.len()); - for mut draft in drafts { - if let Some(snapshot_id) = draft.snapshot_id { - if protected_ref_snapshot_ids.contains(&snapshot_id) { - draft.reasons.insert(TableSnapshotExpirationReason::ProtectedSnapshotRef); - } - if user_defined_ref_snapshot_ids.contains(&snapshot_id) { - draft.reasons.insert(TableSnapshotExpirationReason::UserDefinedSnapshotRef); - } - if ref_retention_conflict_snapshot_ids.contains(&snapshot_id) { - draft - .reasons - .insert(TableSnapshotExpirationReason::SnapshotRefRetentionConflict); - } - } - if table_retention_property_conflict { - draft - .reasons - .insert(TableSnapshotExpirationReason::TableRetentionPropertyConflict); - } - - let state = if snapshot_expiration_requires_manual_review(&draft.reasons) { - TableSnapshotExpirationSnapshotState::ManualReviewRequired - } else if snapshot_expiration_is_retained(&draft.reasons) { - TableSnapshotExpirationSnapshotState::Retained - } else if let Some(timestamp_ms) = draft.timestamp_ms { - if timestamp_ms <= expiration_watermark_ms { - draft.reasons.insert(TableSnapshotExpirationReason::SnapshotAgeExpired); - TableSnapshotExpirationSnapshotState::ExpirationCandidate - } else { - draft - .reasons - .insert(TableSnapshotExpirationReason::SnapshotAgeWithinRetention); - TableSnapshotExpirationSnapshotState::Retained - } - } else { - draft.reasons.insert(TableSnapshotExpirationReason::MissingSnapshotTimestamp); - TableSnapshotExpirationSnapshotState::ManualReviewRequired - }; - - snapshot_reports.push(TableSnapshotExpirationSnapshotReport { - snapshot_id: draft.snapshot_id, - sequence_number: draft.sequence_number, - timestamp_ms: draft.timestamp_ms, - manifest_list: draft.manifest_list, - state, - reasons: draft.reasons.into_iter().collect(), - }); - } - - let retained_snapshot_count = snapshot_reports - .iter() - .filter(|snapshot| snapshot.state == TableSnapshotExpirationSnapshotState::Retained) - .count(); - let expiration_candidate_count = snapshot_reports - .iter() - .filter(|snapshot| snapshot.state == TableSnapshotExpirationSnapshotState::ExpirationCandidate) - .count(); - let manual_review_count = snapshot_reports - .iter() - .filter(|snapshot| snapshot.state == TableSnapshotExpirationSnapshotState::ManualReviewRequired) - .count(); - - TableSnapshotExpirationReport { - table_bucket: table_bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: entry.table_id.clone(), - current_metadata_location: entry.metadata_location.clone(), - current_snapshot_id, - config, - expiration_watermark_ms, - retained_snapshot_count, - expiration_candidate_count, - manual_review_count, - expired_snapshot_ids: Vec::new(), - committed_metadata_location: None, - snapshot_reports, - } -} - -#[derive(Debug, Clone)] -struct TableCompactionDataFileCandidate { - location: String, - size_bytes: u64, - rewrite_prefix: String, - sort_order_id: Option, -} - -#[derive(Debug, Default)] -struct CompactionManifestPlanning { - candidates: Vec, - row_level_planning: TableRowLevelMaintenancePlanningReport, -} - -struct CompactedParquetFile { - data: Vec, - record_count: u64, -} - -#[derive(Debug, Clone)] -struct CompactedDataFile { - object_key: String, - file_path: String, - file_size_bytes: u64, - record_count: u64, - partition_spec_id: i32, - partition: Vec<(String, apache_avro::types::Value)>, - sort_order_id: Option, - status: i32, - snapshot_id: i64, - sequence_number: i64, - file_sequence_number: i64, -} - -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct ManifestDataFileReference { - pub location: String, - pub content: ManifestDataFileContent, - pub object_kind: TableMetadataMaintenanceObjectKind, - pub entry_status: Option, - pub snapshot_id: Option, - pub sequence_number: Option, - pub file_sequence_number: Option, - pub record_count: Option, - pub file_size_bytes: Option, - pub partition: Vec<(String, apache_avro::types::Value)>, - pub sort_order_id: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ManifestDataFileContent { - Data, - PositionDelete, - EqualityDelete, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct ManifestListReference { - pub manifest_path: String, - pub partition_spec_id: Option, - pub sequence_number: Option, - pub added_snapshot_id: Option, -} - -struct CompactionManifestListSummary<'a> { - manifest_path: &'a str, - manifest_length: u64, - partition_spec_id: i32, - snapshot_id: i64, - sequence_number: i64, - added_files_count: usize, - existing_files_count: usize, - added_rows_count: u64, - existing_rows_count: u64, -} - -async fn table_compaction_planning_report( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - entry: &TableEntry, - current_metadata: &serde_json::Value, - config: TableCompactionPlanningConfig, -) -> TableCatalogStoreResult -where - B: TableCatalogObjectBackend, -{ - let current_snapshot_id = current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64); - let warehouse_object_prefix = table_warehouse_object_prefix(entry).ok(); - let mut snapshot_reports = Vec::new(); - let mut candidates = Vec::new(); - let mut rewrite_groups = Vec::new(); - let mut row_level_planning = TableRowLevelMaintenancePlanningReport::default(); - - if let Some(current_snapshot_id) = current_snapshot_id { - let current_snapshot = current_metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - .find(|snapshot| { - snapshot - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_some_and(|snapshot_id| snapshot_id == current_snapshot_id) - }); - match current_snapshot { - Some(snapshot) => { - let manifest_list = snapshot - .get("manifest-list") - .and_then(serde_json::Value::as_str) - .map(ToString::to_string); - match manifest_list.as_deref() { - Some(manifest_list) => { - let planning = match compaction_data_file_candidates( - backend, - table_bucket, - namespace, - table, - warehouse_object_prefix.as_deref(), - manifest_list, - &config, - ) - .await - { - Ok(planning) => planning, - Err(_) => { - snapshot_reports.push(TableCompactionSnapshotReport { - snapshot_id: Some(current_snapshot_id), - manifest_list: Some(manifest_list.to_string()), - status: TableCompactionPlanningStatus::ManualReviewRequired, - reasons: vec![ - TableCompactionPlanningReason::ManifestList, - TableCompactionPlanningReason::ManifestAvroReaderUnavailable, - ], - }); - CompactionManifestPlanning::default() - } - }; - row_level_planning = planning.row_level_planning; - candidates = planning.candidates; - if row_level_planning.status == TableRowLevelMaintenancePlanningStatus::ManualReviewRequired - && snapshot_reports.is_empty() - { - snapshot_reports.push(TableCompactionSnapshotReport { - snapshot_id: Some(current_snapshot_id), - manifest_list: Some(manifest_list.to_string()), - status: TableCompactionPlanningStatus::ManualReviewRequired, - reasons: compaction_row_level_planning_reasons(&row_level_planning), - }); - } else if !candidates.is_empty() && snapshot_reports.is_empty() { - rewrite_groups = compaction_rewrite_groups(&candidates, &config); - let (status, reasons) = if rewrite_groups.is_empty() { - ( - TableCompactionPlanningStatus::NoCandidates, - vec![ - TableCompactionPlanningReason::ManifestList, - TableCompactionPlanningReason::ManifestFile, - ], - ) - } else { - ( - TableCompactionPlanningStatus::RewriteCandidates, - vec![ - TableCompactionPlanningReason::ManifestList, - TableCompactionPlanningReason::ManifestFile, - TableCompactionPlanningReason::SmallDataFile, - TableCompactionPlanningReason::RewriteGroup, - ], - ) - }; - snapshot_reports.push(TableCompactionSnapshotReport { - snapshot_id: Some(current_snapshot_id), - manifest_list: Some(manifest_list.to_string()), - status, - reasons, - }); - } - } - None => snapshot_reports.push(TableCompactionSnapshotReport { - snapshot_id: Some(current_snapshot_id), - manifest_list: None, - status: TableCompactionPlanningStatus::ManualReviewRequired, - reasons: vec![TableCompactionPlanningReason::MissingManifestList], - }), - } - } - None => snapshot_reports.push(TableCompactionSnapshotReport { - snapshot_id: Some(current_snapshot_id), - manifest_list: None, - status: TableCompactionPlanningStatus::ManualReviewRequired, - reasons: vec![TableCompactionPlanningReason::MissingCurrentSnapshot], - }), - } - } - - let manual_review_count = snapshot_reports - .iter() - .filter(|snapshot| snapshot.status == TableCompactionPlanningStatus::ManualReviewRequired) - .count(); - let status = if manual_review_count > 0 { - TableCompactionPlanningStatus::ManualReviewRequired - } else if rewrite_groups.is_empty() { - TableCompactionPlanningStatus::NoCandidates - } else { - TableCompactionPlanningStatus::RewriteCandidates - }; - - Ok(TableCompactionPlanningReport { - table_bucket: table_bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: entry.table_id.clone(), - current_metadata_location: entry.metadata_location.clone(), - current_snapshot_id, - config, - status, - candidate_file_count: candidates.len(), - rewrite_group_count: rewrite_groups.len(), - manual_review_count, - committed_metadata_location: None, - row_level_planning, - rewrite_groups, - snapshot_reports, - }) -} - -async fn compaction_data_file_candidates( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - manifest_list: &str, - config: &TableCompactionPlanningConfig, -) -> TableCatalogStoreResult -where - B: TableCatalogObjectBackend, -{ - let Some(manifest_list_key) = table_catalog_object_key_from_location(table_bucket, manifest_list) else { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest list must be inside the table bucket".to_string(), - )); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &manifest_list_key) - != Some(TableMetadataMaintenanceObjectKind::ManifestList) - { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest list must be inside the table metadata directory".to_string(), - )); - } - let Some(manifest_list_object) = backend.read_object(table_bucket, &manifest_list_key).await? else { - return Err(TableCatalogStoreError::NotFound(format!("compaction manifest list {manifest_list_key}"))); - }; - let manifest_paths = manifest_paths_from_manifest_list_avro(&manifest_list_object.data)?; - let mut planning = CompactionManifestPlanning::default(); - for manifest_location in manifest_paths { - let Some(manifest_key) = table_catalog_object_key_from_location(table_bucket, &manifest_location) else { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest must be inside the table bucket".to_string(), - )); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &manifest_key) - != Some(TableMetadataMaintenanceObjectKind::ManifestFile) - { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest must be inside the table metadata directory".to_string(), - )); - } - let Some(manifest_object) = backend.read_object(table_bucket, &manifest_key).await? else { - return Err(TableCatalogStoreError::NotFound(format!("compaction manifest {manifest_key}"))); - }; - for reference in data_file_references_from_manifest_avro(&manifest_object.data)? { - if reference.object_kind != TableMetadataMaintenanceObjectKind::DataFile { - record_compaction_row_level_delete_file( - backend, - table_bucket, - namespace, - table, - warehouse_object_prefix, - &mut planning.row_level_planning, - &reference, - ) - .await?; - continue; - } - validate_compaction_manifest_entry_status(reference.entry_status)?; - let Some(data_key) = table_catalog_object_key_from_location(table_bucket, &reference.location) else { - return Err(TableCatalogStoreError::Invalid( - "compaction data file must be inside the table bucket".to_string(), - )); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &data_key) - != Some(TableMetadataMaintenanceObjectKind::DataFile) - { - return Err(TableCatalogStoreError::Invalid( - "compaction data file must be inside the table data directory".to_string(), - )); - } - let Some(data_object) = backend.read_object(table_bucket, &data_key).await? else { - return Err(TableCatalogStoreError::NotFound(format!("compaction data file {data_key}"))); - }; - let size_bytes = u64::try_from(data_object.data.len()).unwrap_or(u64::MAX); - if size_bytes <= config.small_file_threshold_bytes { - planning.candidates.push(TableCompactionDataFileCandidate { - rewrite_prefix: compaction_data_file_rewrite_prefix(namespace, table, warehouse_object_prefix, &data_key) - .unwrap_or_else(|| data_key.clone()), - location: data_key, - size_bytes, - sort_order_id: reference.sort_order_id, - }); - } - } - } - Ok(planning) -} - -async fn record_compaction_row_level_delete_file( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - planning: &mut TableRowLevelMaintenancePlanningReport, - reference: &ManifestDataFileReference, -) -> TableCatalogStoreResult<()> -where - B: TableCatalogObjectBackend, -{ - let Some(content) = row_level_delete_file_content(reference.content) else { - return Ok(()); - }; - let Some(delete_key) = table_catalog_object_key_from_location(table_bucket, &reference.location) else { - return Err(TableCatalogStoreError::Invalid( - "compaction delete file must be inside the table bucket".to_string(), - )); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &delete_key) - != Some(TableMetadataMaintenanceObjectKind::DeleteFile) - { - return Err(TableCatalogStoreError::Invalid( - "compaction delete file must be inside the table delete directory".to_string(), - )); - } - - let object_exists = backend.read_object(table_bucket, &delete_key).await?.is_some(); - planning.status = TableRowLevelMaintenancePlanningStatus::ManualReviewRequired; - planning.delete_file_count = planning.delete_file_count.saturating_add(1); - planning.manual_review_count = planning.manual_review_count.saturating_add(1); - push_row_level_planning_reason(planning, TableRowLevelMaintenancePlanningReason::DeleteFileRewriteUnsupported); - match content { - TableRowLevelDeleteFileContent::PositionDelete => { - planning.position_delete_file_count = planning.position_delete_file_count.saturating_add(1); - push_row_level_planning_reason(planning, TableRowLevelMaintenancePlanningReason::PositionDeleteFile); - } - TableRowLevelDeleteFileContent::EqualityDelete => { - planning.equality_delete_file_count = planning.equality_delete_file_count.saturating_add(1); - push_row_level_planning_reason(planning, TableRowLevelMaintenancePlanningReason::EqualityDeleteFile); - } - } - if !object_exists { - push_row_level_planning_reason(planning, TableRowLevelMaintenancePlanningReason::MissingDeleteFile); - } - planning.delete_files.push(TableRowLevelDeleteFilePlanningReport { - file_location: delete_key, - content, - object_exists, - record_count: reference.record_count, - file_size_bytes: reference.file_size_bytes, - sequence_number: reference.sequence_number, - file_sequence_number: reference.file_sequence_number, - }); - Ok(()) -} - -fn row_level_delete_file_content(content: ManifestDataFileContent) -> Option { - match content { - ManifestDataFileContent::Data => None, - ManifestDataFileContent::PositionDelete => Some(TableRowLevelDeleteFileContent::PositionDelete), - ManifestDataFileContent::EqualityDelete => Some(TableRowLevelDeleteFileContent::EqualityDelete), - } -} - -fn push_row_level_planning_reason( - planning: &mut TableRowLevelMaintenancePlanningReport, - reason: TableRowLevelMaintenancePlanningReason, -) { - if !planning.reasons.contains(&reason) { - planning.reasons.push(reason); - } -} - -fn compaction_row_level_planning_reasons( - planning: &TableRowLevelMaintenancePlanningReport, -) -> Vec { - let mut reasons = vec![ - TableCompactionPlanningReason::ManifestList, - TableCompactionPlanningReason::ManifestFile, - TableCompactionPlanningReason::DeleteFile, - TableCompactionPlanningReason::RowLevelRewriteUnsupported, - ]; - if planning.position_delete_file_count > 0 { - reasons.push(TableCompactionPlanningReason::PositionDeleteFile); - } - if planning.equality_delete_file_count > 0 { - reasons.push(TableCompactionPlanningReason::EqualityDeleteFile); - } - reasons -} - -async fn compaction_current_data_files( - backend: &B, - table_bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - entry: &TableEntry, - current_metadata: &serde_json::Value, -) -> TableCatalogStoreResult> -where - B: TableCatalogObjectBackend, -{ - let current_snapshot_id = current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| TableCatalogStoreError::Invalid("compaction requires current snapshot metadata".to_string()))?; - let current_snapshot = current_metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - .find(|snapshot| { - snapshot - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_some_and(|snapshot_id| snapshot_id == current_snapshot_id) - }) - .ok_or_else(|| TableCatalogStoreError::Invalid("compaction requires current snapshot entry".to_string()))?; - let manifest_list = current_snapshot - .get("manifest-list") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| TableCatalogStoreError::Invalid("compaction requires current snapshot manifest list".to_string()))?; - - let warehouse_object_prefix = table_warehouse_object_prefix(entry).ok(); - let Some(manifest_list_key) = table_catalog_object_key_from_location(table_bucket, manifest_list) else { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest list must be inside the table bucket".to_string(), - )); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix.as_deref(), &manifest_list_key) - != Some(TableMetadataMaintenanceObjectKind::ManifestList) - { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest list must be inside the table metadata directory".to_string(), - )); - } - let Some(manifest_list_object) = backend.read_object(table_bucket, &manifest_list_key).await? else { - return Err(TableCatalogStoreError::NotFound(format!("compaction manifest list {manifest_list_key}"))); - }; - - let mut data_files = Vec::new(); - for manifest_reference in manifest_list_references_from_manifest_list_avro(&manifest_list_object.data)? { - let Some(manifest_key) = table_catalog_object_key_from_location(table_bucket, &manifest_reference.manifest_path) else { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest must be inside the table bucket".to_string(), - )); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix.as_deref(), &manifest_key) - != Some(TableMetadataMaintenanceObjectKind::ManifestFile) - { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest must be inside the table metadata directory".to_string(), - )); - } - let Some(manifest_object) = backend.read_object(table_bucket, &manifest_key).await? else { - return Err(TableCatalogStoreError::NotFound(format!("compaction manifest {manifest_key}"))); - }; - for reference in data_file_references_from_manifest_avro(&manifest_object.data)? { - if reference.object_kind != TableMetadataMaintenanceObjectKind::DataFile { - return Err(TableCatalogStoreError::Invalid( - "compaction currently does not support delete files".to_string(), - )); - } - validate_compaction_manifest_entry_status(reference.entry_status)?; - let Some(data_key) = table_catalog_object_key_from_location(table_bucket, &reference.location) else { - return Err(TableCatalogStoreError::Invalid( - "compaction data file must be inside the table bucket".to_string(), - )); - }; - if table_maintenance_object_kind(namespace, table, warehouse_object_prefix.as_deref(), &data_key) - != Some(TableMetadataMaintenanceObjectKind::DataFile) - { - return Err(TableCatalogStoreError::Invalid( - "compaction data file must be inside the table data directory".to_string(), - )); - } - let Some(data_object) = backend.read_object(table_bucket, &data_key).await? else { - return Err(TableCatalogStoreError::NotFound(format!("compaction data file {data_key}"))); - }; - let snapshot_id = reference - .snapshot_id - .or(manifest_reference.added_snapshot_id) - .ok_or_else(|| { - TableCatalogStoreError::Invalid("compaction manifest data file missing snapshot id".to_string()) - })?; - let sequence_number = reference - .sequence_number - .or(manifest_reference.sequence_number) - .ok_or_else(|| { - TableCatalogStoreError::Invalid("compaction manifest data file missing sequence number".to_string()) - })?; - let file_sequence_number = reference - .file_sequence_number - .or(manifest_reference.sequence_number) - .ok_or_else(|| { - TableCatalogStoreError::Invalid("compaction manifest data file missing file sequence number".to_string()) - })?; - data_files.push(CompactedDataFile { - object_key: data_key, - file_path: reference.location, - file_size_bytes: reference - .file_size_bytes - .unwrap_or_else(|| u64::try_from(data_object.data.len()).unwrap_or(u64::MAX)), - record_count: match reference.record_count { - Some(record_count) => record_count, - None => parquet_record_count(&data_object.data)?, - }, - partition_spec_id: manifest_reference.partition_spec_id.unwrap_or(0), - partition: reference.partition, - sort_order_id: reference.sort_order_id, - status: 0, - snapshot_id, - sequence_number, - file_sequence_number, - }); - } - } - - Ok(data_files) -} - -fn compaction_rewrite_groups( - candidates: &[TableCompactionDataFileCandidate], - config: &TableCompactionPlanningConfig, -) -> Vec { - let mut groups = Vec::new(); - let mut candidates_by_prefix = BTreeMap::<(&str, Option), Vec<&TableCompactionDataFileCandidate>>::new(); - for candidate in candidates { - candidates_by_prefix - .entry((candidate.rewrite_prefix.as_str(), candidate.sort_order_id)) - .or_default() - .push(candidate); - } - - for ((_, sort_order_id), prefix_candidates) in candidates_by_prefix { - push_compaction_rewrite_groups_for_prefix(&mut groups, prefix_candidates.as_slice(), sort_order_id, config); - } - groups -} - -fn push_compaction_rewrite_groups_for_prefix( - groups: &mut Vec, - candidates: &[&TableCompactionDataFileCandidate], - sort_order_id: Option, - config: &TableCompactionPlanningConfig, -) { - let mut current_locations = Vec::new(); - let mut current_bytes = 0_u64; - for candidate in candidates { - let next_bytes = current_bytes.saturating_add(candidate.size_bytes); - if !current_locations.is_empty() && next_bytes > config.max_rewrite_bytes_per_job { - push_compaction_rewrite_group(groups, &mut current_locations, &mut current_bytes, sort_order_id, config); - } - current_locations.push(candidate.location.clone()); - current_bytes = current_bytes.saturating_add(candidate.size_bytes); - } - push_compaction_rewrite_group(groups, &mut current_locations, &mut current_bytes, sort_order_id, config); -} - -fn compaction_data_file_rewrite_prefix( - namespace: &Namespace, - table: &IdentifierSegment, - warehouse_object_prefix: Option<&str>, - location: &str, -) -> Option { - let warehouse_data_prefix = warehouse_object_prefix - .map(|prefix| format!("{prefix}{DATA_DIR}")) - .unwrap_or_else(|| default_table_data_dir_path(namespace, table)); - let default_data_prefix = format!("{}/", default_table_data_dir_path(namespace, table)); - if let Some(relative_path) = location.strip_prefix(&default_data_prefix) { - return Some(compaction_data_file_output_prefix(&warehouse_data_prefix, relative_path)); - } - if let Some(warehouse_object_prefix) = warehouse_object_prefix { - let warehouse_input_prefix = format!("{warehouse_object_prefix}{DATA_DIR}/"); - if let Some(relative_path) = location.strip_prefix(&warehouse_input_prefix) { - return Some(compaction_data_file_output_prefix(&warehouse_data_prefix, relative_path)); - } - } - None -} - -fn compaction_data_file_output_prefix(output_data_prefix: &str, relative_path: &str) -> String { - relative_path - .rsplit_once('/') - .map(|(partition_path, _)| format!("{output_data_prefix}/{partition_path}")) - .unwrap_or_else(|| output_data_prefix.to_string()) -} - -fn push_compaction_rewrite_group( - groups: &mut Vec, - current_locations: &mut Vec, - current_bytes: &mut u64, - sort_order_id: Option, - config: &TableCompactionPlanningConfig, -) { - if current_locations.len() >= config.min_input_files { - let input_file_count = current_locations.len(); - groups.push(TableCompactionRewriteGroup { - group_id: format!("{:04}", groups.len() + 1), - sort_order_id, - input_file_locations: std::mem::take(current_locations), - input_file_count, - input_bytes: *current_bytes, - output_file_location: None, - output_bytes: None, - }); - } else { - current_locations.clear(); - } - *current_bytes = 0; -} - -fn compaction_rewrite_group_partition( - data_files_by_key: &BTreeMap<&str, &CompactedDataFile>, - rewrite_group: &TableCompactionRewriteGroup, -) -> TableCatalogStoreResult<(i32, Vec<(String, apache_avro::types::Value)>)> { - let mut partition_spec_id = None; - let mut partition = None; - for input in &rewrite_group.input_file_locations { - let Some(data_file) = data_files_by_key.get(input.as_str()) else { - return Err(TableCatalogStoreError::Invalid( - "compaction rewrite input is missing from current manifest".to_string(), - )); - }; - match (partition_spec_id, partition.as_ref()) { - (None, None) => { - partition_spec_id = Some(data_file.partition_spec_id); - partition = Some(data_file.partition.clone()); - } - (Some(expected_spec_id), Some(expected_partition)) - if expected_spec_id == data_file.partition_spec_id && expected_partition == &data_file.partition => {} - _ => { - return Err(TableCatalogStoreError::Invalid( - "compaction rewrite group must contain a single partition tuple".to_string(), - )); - } - } - } - Ok((partition_spec_id.unwrap_or(0), partition.unwrap_or_default())) -} - -fn compaction_rewrite_group_sort_order( - data_files_by_key: &BTreeMap<&str, &CompactedDataFile>, - rewrite_group: &TableCompactionRewriteGroup, -) -> TableCatalogStoreResult> { - let mut sort_order_id = None; - let mut initialized = false; - for input in &rewrite_group.input_file_locations { - let Some(data_file) = data_files_by_key.get(input.as_str()) else { - return Err(TableCatalogStoreError::Invalid( - "compaction rewrite input is missing from current manifest".to_string(), - )); - }; - if !initialized { - sort_order_id = data_file.sort_order_id; - initialized = true; - } else if sort_order_id != data_file.sort_order_id { - return Err(TableCatalogStoreError::Invalid( - "compaction rewrite group must contain a single sort order".to_string(), - )); - } - } - if rewrite_group.sort_order_id != sort_order_id { - return Err(TableCatalogStoreError::Invalid( - "compaction rewrite group sort order changed after planning".to_string(), - )); - } - Ok(sort_order_id) -} - -fn compaction_manifest_partition_spec_id(data_files: &[CompactedDataFile]) -> TableCatalogStoreResult { - let Some(first) = data_files.first() else { - return Ok(0); - }; - let partition_spec_id = first.partition_spec_id; - if data_files - .iter() - .any(|data_file| data_file.partition_spec_id != partition_spec_id) - { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest cannot mix partition spec ids".to_string(), - )); - } - Ok(partition_spec_id) -} - -fn compact_parquet_data_files(input_files: &[(String, Vec)]) -> TableCatalogStoreResult { - let mut schema: Option = None; - let mut batches = Vec::new(); - let mut record_count = 0_u64; - - for (location, data) in input_files { - let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(data.clone())).map_err(|err| { - TableCatalogStoreError::Invalid(format!("failed to read compaction input parquet {location}: {err}")) - })?; - let file_schema = builder.schema().clone(); - match schema.as_ref() { - Some(expected_schema) if expected_schema.as_ref() != file_schema.as_ref() => { - return Err(TableCatalogStoreError::Invalid("compaction input parquet schemas must match".to_string())); - } - Some(_) => {} - None => schema = Some(file_schema), - } - - let reader = builder.build().map_err(|err| { - TableCatalogStoreError::Invalid(format!("failed to build compaction parquet reader {location}: {err}")) - })?; - for batch in reader { - let batch = batch.map_err(|err| { - TableCatalogStoreError::Invalid(format!("failed to read compaction parquet batch {location}: {err}")) - })?; - record_count = record_count.saturating_add(u64::try_from(batch.num_rows()).unwrap_or(u64::MAX)); - batches.push(batch); - } - } - - let Some(schema) = schema else { - return Err(TableCatalogStoreError::Invalid( - "compaction requires at least one parquet input file".to_string(), - )); - }; - let mut data = Vec::new(); - { - let mut writer = ArrowWriter::try_new(&mut data, schema, None) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to build compaction parquet writer: {err}")))?; - for batch in batches { - writer - .write(&batch) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to write compaction parquet batch: {err}")))?; - } - writer - .close() - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to close compaction parquet writer: {err}")))?; - } - - Ok(CompactedParquetFile { data, record_count }) -} - -fn validate_compaction_manifest_entry_status(entry_status: Option) -> TableCatalogStoreResult<()> { - match entry_status { - Some(0 | 1) => Ok(()), - Some(2) => Err(TableCatalogStoreError::Invalid( - "compaction currently does not support deleted manifest entries".to_string(), - )), - Some(_) => Err(TableCatalogStoreError::Invalid( - "compaction manifest entry status is unsupported".to_string(), - )), - None => Err(TableCatalogStoreError::Invalid("compaction manifest entry missing status".to_string())), - } -} - -fn parquet_record_count(data: &[u8]) -> TableCatalogStoreResult { - let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::copy_from_slice(data)) - .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read compaction parquet metadata: {err}")))?; - u64::try_from(builder.metadata().file_metadata().num_rows()) - .map_err(|_| TableCatalogStoreError::Invalid("compaction parquet record count must not be negative".to_string())) -} - -fn compacted_manifest_list_avro_bytes(summary: CompactionManifestListSummary<'_>) -> TableCatalogStoreResult> { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "manifest_file", - "fields": [ - {"name": "manifest_path", "type": "string"}, - {"name": "manifest_length", "type": "long"}, - {"name": "partition_spec_id", "type": "int"}, - {"name": "content", "type": "int"}, - {"name": "sequence_number", "type": "long"}, - {"name": "min_sequence_number", "type": "long"}, - {"name": "added_snapshot_id", "type": "long"}, - {"name": "added_files_count", "type": "int"}, - {"name": "existing_files_count", "type": "int"}, - {"name": "deleted_files_count", "type": "int"}, - {"name": "added_rows_count", "type": "long"}, - {"name": "existing_rows_count", "type": "long"}, - {"name": "deleted_rows_count", "type": "long"}, - {"name": "partitions", "type": ["null", {"type": "array", "items": {"type": "record", "name": "field_summary", "fields": [ - {"name": "contains_null", "type": "boolean"}, - {"name": "lower_bound", "type": ["null", "bytes"], "default": null}, - {"name": "upper_bound", "type": ["null", "bytes"], "default": null} - ]}}], "default": null} - ] - } - "#, - ) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to build compaction manifest list schema: {err}")))?; - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - writer - .append(apache_avro::types::Value::Record(vec![ - ( - "manifest_path".to_string(), - apache_avro::types::Value::String(summary.manifest_path.to_string()), - ), - ( - "manifest_length".to_string(), - apache_avro::types::Value::Long(i64::try_from(summary.manifest_length).unwrap_or(i64::MAX)), - ), - ("partition_spec_id".to_string(), apache_avro::types::Value::Int(summary.partition_spec_id)), - ("content".to_string(), apache_avro::types::Value::Int(0)), - ("sequence_number".to_string(), apache_avro::types::Value::Long(summary.sequence_number)), - ( - "min_sequence_number".to_string(), - apache_avro::types::Value::Long(summary.sequence_number), - ), - ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(summary.snapshot_id)), - ( - "added_files_count".to_string(), - apache_avro::types::Value::Int(i32::try_from(summary.added_files_count).unwrap_or(i32::MAX)), - ), - ( - "existing_files_count".to_string(), - apache_avro::types::Value::Int(i32::try_from(summary.existing_files_count).unwrap_or(i32::MAX)), - ), - ("deleted_files_count".to_string(), apache_avro::types::Value::Int(0)), - ( - "added_rows_count".to_string(), - apache_avro::types::Value::Long(i64::try_from(summary.added_rows_count).unwrap_or(i64::MAX)), - ), - ( - "existing_rows_count".to_string(), - apache_avro::types::Value::Long(i64::try_from(summary.existing_rows_count).unwrap_or(i64::MAX)), - ), - ("deleted_rows_count".to_string(), apache_avro::types::Value::Long(0)), - ( - "partitions".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ])) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to write compaction manifest list: {err}")))?; - writer - .into_inner() - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to flush compaction manifest list: {err}"))) -} - -fn compacted_manifest_avro_schema(data_files: &[CompactedDataFile]) -> TableCatalogStoreResult { - let partition_fields = compaction_partition_schema_fields(data_files)?; - let partition_schema_fields = partition_fields - .into_iter() - .map(|(name, field_type)| { - serde_json::json!({ - "name": name, - "type": field_type - }) - }) - .collect::>(); - let schema = serde_json::json!({ - "type": "record", - "name": "manifest_entry", - "fields": [ - {"name": "status", "type": "int"}, - {"name": "snapshot_id", "type": "long"}, - {"name": "sequence_number", "type": "long"}, - {"name": "file_sequence_number", "type": "long"}, - { - "name": "data_file", - "type": { - "type": "record", - "name": "data_file", - "fields": [ - {"name": "content", "type": "int"}, - {"name": "file_path", "type": "string"}, - {"name": "file_format", "type": "string"}, - { - "name": "partition", - "type": { - "type": "record", - "name": "partition", - "fields": partition_schema_fields - } - }, - {"name": "record_count", "type": "long"}, - {"name": "file_size_in_bytes", "type": "long"}, - {"name": "column_sizes", "type": ["null", {"type": "map", "values": "long"}], "default": null}, - {"name": "value_counts", "type": ["null", {"type": "map", "values": "long"}], "default": null}, - {"name": "null_value_counts", "type": ["null", {"type": "map", "values": "long"}], "default": null}, - {"name": "nan_value_counts", "type": ["null", {"type": "map", "values": "long"}], "default": null}, - {"name": "lower_bounds", "type": ["null", {"type": "map", "values": "bytes"}], "default": null}, - {"name": "upper_bounds", "type": ["null", {"type": "map", "values": "bytes"}], "default": null}, - {"name": "key_metadata", "type": ["null", "bytes"], "default": null}, - {"name": "split_offsets", "type": ["null", {"type": "array", "items": "long"}], "default": null}, - {"name": "equality_ids", "type": ["null", {"type": "array", "items": "int"}], "default": null}, - {"name": "sort_order_id", "type": ["null", "int"], "default": null} - ] - } - } - ] - }); - apache_avro::Schema::parse_str(&schema.to_string()) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to build compaction manifest schema: {err}"))) -} - -fn compaction_partition_schema_fields( - data_files: &[CompactedDataFile], -) -> TableCatalogStoreResult> { - let Some(first) = data_files.first() else { - return Ok(Vec::new()); - }; - let mut expected = Vec::with_capacity(first.partition.len()); - for (field_name, field_value) in &first.partition { - let Some(field_type) = compaction_partition_field_schema(field_value) else { - return Err(TableCatalogStoreError::Invalid( - "compaction partition value type is unsupported".to_string(), - )); - }; - expected.push((field_name.clone(), field_type)); - } - - for data_file in data_files.iter().skip(1) { - if data_file.partition.len() != expected.len() { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest partition schemas must match".to_string(), - )); - } - for ((expected_name, expected_type), (field_name, field_value)) in expected.iter().zip(&data_file.partition) { - let Some(field_type) = compaction_partition_field_schema(field_value) else { - return Err(TableCatalogStoreError::Invalid( - "compaction partition value type is unsupported".to_string(), - )); - }; - if expected_name != field_name || expected_type != &field_type { - return Err(TableCatalogStoreError::Invalid( - "compaction manifest partition schemas must match".to_string(), - )); - } - } - } - Ok(expected) -} - -fn compaction_partition_field_schema(value: &apache_avro::types::Value) -> Option { - match avro_non_union_value(value) { - apache_avro::types::Value::Boolean(_) => Some(serde_json::json!("boolean")), - apache_avro::types::Value::Int(_) => Some(serde_json::json!("int")), - apache_avro::types::Value::Long(_) => Some(serde_json::json!("long")), - apache_avro::types::Value::Float(_) => Some(serde_json::json!("float")), - apache_avro::types::Value::Double(_) => Some(serde_json::json!("double")), - apache_avro::types::Value::Bytes(_) => Some(serde_json::json!("bytes")), - apache_avro::types::Value::String(_) => Some(serde_json::json!("string")), - apache_avro::types::Value::Date(_) => Some(serde_json::json!({"type": "int", "logicalType": "date"})), - apache_avro::types::Value::TimeMillis(_) => Some(serde_json::json!({"type": "int", "logicalType": "time-millis"})), - apache_avro::types::Value::TimeMicros(_) => Some(serde_json::json!({"type": "long", "logicalType": "time-micros"})), - apache_avro::types::Value::TimestampMillis(_) => { - Some(serde_json::json!({"type": "long", "logicalType": "timestamp-millis"})) - } - apache_avro::types::Value::TimestampMicros(_) => { - Some(serde_json::json!({"type": "long", "logicalType": "timestamp-micros"})) - } - apache_avro::types::Value::LocalTimestampMillis(_) => { - Some(serde_json::json!({"type": "long", "logicalType": "local-timestamp-millis"})) - } - apache_avro::types::Value::LocalTimestampMicros(_) => { - Some(serde_json::json!({"type": "long", "logicalType": "local-timestamp-micros"})) - } - apache_avro::types::Value::Uuid(_) => Some(serde_json::json!({"type": "string", "logicalType": "uuid"})), - _ => None, - } -} - -fn compacted_manifest_avro_bytes(data_files: &[CompactedDataFile]) -> TableCatalogStoreResult> { - let schema = compacted_manifest_avro_schema(data_files)?; - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - for data_file in data_files { - let sort_order_id = match data_file.sort_order_id { - Some(sort_order_id) => apache_avro::types::Value::Union(1, Box::new(apache_avro::types::Value::Int(sort_order_id))), - None => apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - }; - writer - .append(apache_avro::types::Value::Record(vec![ - ("status".to_string(), apache_avro::types::Value::Int(data_file.status)), - ("snapshot_id".to_string(), apache_avro::types::Value::Long(data_file.snapshot_id)), - ("sequence_number".to_string(), apache_avro::types::Value::Long(data_file.sequence_number)), - ( - "file_sequence_number".to_string(), - apache_avro::types::Value::Long(data_file.file_sequence_number), - ), - ( - "data_file".to_string(), - apache_avro::types::Value::Record(vec![ - ("content".to_string(), apache_avro::types::Value::Int(0)), - ("file_path".to_string(), apache_avro::types::Value::String(data_file.file_path.clone())), - ("file_format".to_string(), apache_avro::types::Value::String("PARQUET".to_string())), - ("partition".to_string(), apache_avro::types::Value::Record(data_file.partition.clone())), - ( - "record_count".to_string(), - apache_avro::types::Value::Long(i64::try_from(data_file.record_count).unwrap_or(i64::MAX)), - ), - ( - "file_size_in_bytes".to_string(), - apache_avro::types::Value::Long(i64::try_from(data_file.file_size_bytes).unwrap_or(i64::MAX)), - ), - ( - "column_sizes".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ( - "value_counts".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ( - "null_value_counts".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ( - "nan_value_counts".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ( - "lower_bounds".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ( - "upper_bounds".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ( - "key_metadata".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ( - "split_offsets".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ( - "equality_ids".to_string(), - apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), - ), - ("sort_order_id".to_string(), sort_order_id), - ]), - ), - ])) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to write compaction manifest: {err}")))?; - } - writer - .into_inner() - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to flush compaction manifest: {err}"))) -} - -fn compaction_snapshot_id(current_metadata: &serde_json::Value, entry: &TableEntry, now: OffsetDateTime) -> i64 { - let generation = i64::try_from(entry.generation).unwrap_or(i64::MAX); - let mut snapshot_id = unix_timestamp_millis(now).saturating_mul(1000).saturating_add(generation); - let existing_snapshot_ids = current_metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - .filter_map(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64)) - .collect::>(); - while existing_snapshot_ids.contains(&snapshot_id) { - snapshot_id = snapshot_id.saturating_add(1); - } - snapshot_id -} - -fn next_compaction_sequence_number(current_metadata: &serde_json::Value) -> i64 { - current_metadata - .get("last-sequence-number") - .and_then(serde_json::Value::as_i64) - .unwrap_or(0) - .saturating_add(1) -} - -fn compaction_metadata_json( - current_metadata: &serde_json::Value, - entry: &TableEntry, - snapshot_id: i64, - sequence_number: i64, - manifest_list: &str, - previous_metadata_location: &str, - now: OffsetDateTime, -) -> TableCatalogStoreResult> { - let mut metadata = current_metadata.clone(); - let now_ms = unix_timestamp_millis(now); - let Some(metadata_object) = metadata.as_object_mut() else { - return Err(TableCatalogStoreError::Invalid( - "compaction metadata source must be a JSON object".to_string(), - )); - }; - metadata_object.insert("last-sequence-number".to_string(), serde_json::json!(sequence_number)); - metadata_object.insert("last-updated-ms".to_string(), serde_json::json!(now_ms)); - metadata_object.insert("current-snapshot-id".to_string(), serde_json::json!(snapshot_id)); - - let snapshots = metadata_object - .entry("snapshots".to_string()) - .or_insert_with(|| serde_json::json!([])); - let Some(snapshots) = snapshots.as_array_mut() else { - return Err(TableCatalogStoreError::Invalid( - "compaction metadata snapshots must be an array".to_string(), - )); - }; - snapshots.push(serde_json::json!({ - "snapshot-id": snapshot_id, - "sequence-number": sequence_number, - "timestamp-ms": now_ms, - "manifest-list": manifest_list, - "summary": { - "operation": "rewrite", - "rustfs.maintenance": "compaction" - } - })); - - let snapshot_log = metadata_object - .entry("snapshot-log".to_string()) - .or_insert_with(|| serde_json::json!([])); - let Some(snapshot_log) = snapshot_log.as_array_mut() else { - return Err(TableCatalogStoreError::Invalid( - "compaction metadata snapshot log must be an array".to_string(), - )); - }; - snapshot_log.push(serde_json::json!({ - "timestamp-ms": now_ms, - "snapshot-id": snapshot_id - })); - - let metadata_log = metadata_object - .entry("metadata-log".to_string()) - .or_insert_with(|| serde_json::json!([])); - let Some(metadata_log) = metadata_log.as_array_mut() else { - return Err(TableCatalogStoreError::Invalid("compaction metadata log must be an array".to_string())); - }; - metadata_log.push(serde_json::json!({ - "timestamp-ms": now_ms, - "metadata-file": previous_metadata_location - })); - - let refs = metadata_object - .entry("refs".to_string()) - .or_insert_with(|| serde_json::json!({})); - let Some(refs) = refs.as_object_mut() else { - return Err(TableCatalogStoreError::Invalid("compaction metadata refs must be an object".to_string())); - }; - refs.insert( - ICEBERG_MAIN_REF.to_string(), - serde_json::json!({ - "snapshot-id": snapshot_id, - "type": "branch" - }), - ); - metadata_object - .entry("location".to_string()) - .or_insert_with(|| serde_json::json!(entry.warehouse_location)); - - serde_json::to_vec(&metadata) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize compaction metadata: {err}"))) -} - -fn snapshot_expiration_drafts( - current_metadata: &serde_json::Value, - current_snapshot_id: Option, -) -> Vec { - let Some(snapshots) = current_metadata.get("snapshots").and_then(serde_json::Value::as_array) else { - return Vec::new(); - }; - - snapshots - .iter() - .map(|snapshot| { - let snapshot_id = snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64); - let timestamp_ms = snapshot.get("timestamp-ms").and_then(serde_json::Value::as_i64); - let mut reasons = BTreeSet::new(); - if snapshot_id.is_none() { - reasons.insert(TableSnapshotExpirationReason::MissingSnapshotId); - } - if timestamp_ms.is_none() { - reasons.insert(TableSnapshotExpirationReason::MissingSnapshotTimestamp); - } - if snapshot_id.is_some() && snapshot_id == current_snapshot_id { - reasons.insert(TableSnapshotExpirationReason::CurrentSnapshot); - } - - TableSnapshotExpirationDraft { - snapshot_id, - sequence_number: snapshot.get("sequence-number").and_then(serde_json::Value::as_i64), - timestamp_ms, - manifest_list: snapshot - .get("manifest-list") - .and_then(serde_json::Value::as_str) - .map(ToString::to_string), - reasons, - } - }) - .collect() -} - -fn mark_recent_snapshots_to_keep(drafts: &mut [TableSnapshotExpirationDraft], min_snapshots_to_keep: usize) { - let mut snapshots_by_time = drafts - .iter() - .enumerate() - .filter_map(|(index, draft)| Some((draft.timestamp_ms?, index))) - .collect::>(); - snapshots_by_time.sort_by(|(left_timestamp, left_index), (right_timestamp, right_index)| { - right_timestamp.cmp(left_timestamp).then_with(|| left_index.cmp(right_index)) - }); - - for (_, index) in snapshots_by_time.into_iter().take(min_snapshots_to_keep) { - drafts[index] - .reasons - .insert(TableSnapshotExpirationReason::MinSnapshotsToKeep); - } -} - -fn snapshot_expiration_ref_state( - current_metadata: &serde_json::Value, - current_snapshot_id: Option, -) -> (BTreeSet, BTreeSet, BTreeSet) { - let mut protected_ref_snapshot_ids = BTreeSet::new(); - let mut user_defined_ref_snapshot_ids = BTreeSet::new(); - let mut ref_retention_conflict_snapshot_ids = BTreeSet::new(); - let Some(refs) = current_metadata.get("refs").and_then(serde_json::Value::as_object) else { - return ( - protected_ref_snapshot_ids, - user_defined_ref_snapshot_ids, - ref_retention_conflict_snapshot_ids, - ); - }; - - for (name, reference) in refs { - let Some(snapshot_id) = reference.get("snapshot-id").and_then(serde_json::Value::as_i64) else { - continue; - }; - if name != ICEBERG_MAIN_REF || Some(snapshot_id) != current_snapshot_id { - protected_ref_snapshot_ids.insert(snapshot_id); - } - if name != ICEBERG_MAIN_REF { - user_defined_ref_snapshot_ids.insert(snapshot_id); - } - if snapshot_ref_has_retention_policy(reference) { - ref_retention_conflict_snapshot_ids.insert(snapshot_id); - } - } - - ( - protected_ref_snapshot_ids, - user_defined_ref_snapshot_ids, - ref_retention_conflict_snapshot_ids, - ) -} - -fn snapshot_ref_has_retention_policy(reference: &serde_json::Value) -> bool { - reference.get(ICEBERG_REF_MIN_SNAPSHOTS_TO_KEEP_FIELD).is_some() - || reference.get(ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD).is_some() - || reference.get(ICEBERG_REF_MAX_REF_AGE_MS_FIELD).is_some() -} - -fn snapshot_expiration_table_property_conflicts( - current_metadata: &serde_json::Value, - config: &TableSnapshotExpirationConfig, -) -> bool { - let Some(properties) = current_metadata.get("properties").and_then(serde_json::Value::as_object) else { - return false; - }; - - if properties.contains_key(ICEBERG_MAX_REF_AGE_MS_PROPERTY) { - return true; - } - if retention_property_conflicts_usize(properties, ICEBERG_MIN_SNAPSHOTS_TO_KEEP_PROPERTY, config.min_snapshots_to_keep) { - return true; - } - retention_property_conflicts_i64(properties, ICEBERG_MAX_SNAPSHOT_AGE_MS_PROPERTY, config.max_snapshot_age_ms) -} - -fn retention_property_conflicts_usize( - properties: &serde_json::Map, - key: &str, - expected: usize, -) -> bool { - let Some(value) = properties.get(key) else { - return false; - }; - serde_json_i64(value).and_then(|value| usize::try_from(value).ok()) != Some(expected) -} - -fn retention_property_conflicts_i64(properties: &serde_json::Map, key: &str, expected: i64) -> bool { - let Some(value) = properties.get(key) else { - return false; - }; - serde_json_i64(value) != Some(expected) -} - -fn serde_json_i64(value: &serde_json::Value) -> Option { - value.as_i64().or_else(|| value.as_str()?.parse::().ok()) -} - -fn snapshot_expiration_requires_manual_review(reasons: &BTreeSet) -> bool { - reasons.contains(&TableSnapshotExpirationReason::MissingSnapshotId) - || reasons.contains(&TableSnapshotExpirationReason::MissingSnapshotTimestamp) - || reasons.contains(&TableSnapshotExpirationReason::UserDefinedSnapshotRef) - || reasons.contains(&TableSnapshotExpirationReason::SnapshotRefRetentionConflict) - || reasons.contains(&TableSnapshotExpirationReason::TableRetentionPropertyConflict) -} - -fn snapshot_expiration_is_retained(reasons: &BTreeSet) -> bool { - reasons.contains(&TableSnapshotExpirationReason::CurrentSnapshot) - || reasons.contains(&TableSnapshotExpirationReason::MinSnapshotsToKeep) - || reasons.contains(&TableSnapshotExpirationReason::ProtectedSnapshotRef) -} - -fn unix_timestamp_millis(now: OffsetDateTime) -> i64 { - now.unix_timestamp() - .saturating_mul(1000) - .saturating_add(i64::from(now.millisecond())) -} - -fn maintenance_timestamp(now: OffsetDateTime) -> String { - now.format(&time::format_description::well_known::Rfc3339) - .unwrap_or_else(|_| now.unix_timestamp().to_string()) -} - -fn default_table_maintenance_worker_lease_timeout_seconds() -> u64 { - TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_DEFAULT_SECONDS -} - -fn parse_maintenance_timestamp(timestamp: &str) -> Option { - OffsetDateTime::parse(timestamp, &time::format_description::well_known::Rfc3339).ok() -} - -fn table_maintenance_quarantine_operator_reason(action: &str, reason: Option<&str>) -> String { - let reason = reason.map(str::trim).filter(|reason| !reason.is_empty()); - match reason { - Some(reason) => format!("maintenance quarantine {action} by operator: {reason}"), - None => format!("maintenance quarantine {action} by operator"), - } -} - -fn push_table_maintenance_audit_event( - report: &mut TableMetadataMaintenanceReport, - timestamp: OffsetDateTime, - actor: TableMaintenanceAuditActor, - action: TableMaintenanceAuditAction, - reason: Option, - before_status: Option, - before_quarantined_object_count: Option, -) { - report.audit_events.push(TableMaintenanceAuditEvent { - timestamp: maintenance_timestamp(timestamp), - actor, - action, - reason, - before_status, - after_status: Some(report.job.status.clone()), - before_quarantined_object_count, - after_quarantined_object_count: Some(report.job.quarantined_object_count), - recommended_actions: report.job.recommended_actions.clone(), - }); -} - -fn table_maintenance_recommended_actions(job: &TableMetadataMaintenanceJob) -> Vec { - let mut actions = Vec::new(); - match job.status { - TableMetadataMaintenanceJobStatus::NotYetRun => {} - TableMetadataMaintenanceJobStatus::Queued => { - actions.push(TableMaintenanceRecommendedAction::RunMaintenanceWorker); - } - TableMetadataMaintenanceJobStatus::Running => { - actions.push(TableMaintenanceRecommendedAction::WaitForActiveWorker); - } - TableMetadataMaintenanceJobStatus::Successful => { - if matches!(job.operation, TableMetadataMaintenanceOperation::DryRun) - && (job.deletable_metadata_file_count > 0 || job.deletable_object_count > 0) - { - actions.push(TableMaintenanceRecommendedAction::ReviewAndRunDelete); - } else { - actions.push(TableMaintenanceRecommendedAction::NoActionRequired); - } - } - TableMetadataMaintenanceJobStatus::Failed => { - if job - .failure_reason - .as_deref() - .is_some_and(|reason| reason == TABLE_MAINTENANCE_DELETE_DISABLED_REASON) - { - actions.push(TableMaintenanceRecommendedAction::EnableDelete); - } - if job.quarantine_enabled && job.quarantined_object_count > 0 { - actions.push(TableMaintenanceRecommendedAction::ReviewQuarantine); - } - if job.next_retry_after.is_some() { - actions.push(TableMaintenanceRecommendedAction::WaitForRetryBackoff); - } - if actions.is_empty() { - actions.push(TableMaintenanceRecommendedAction::InvestigateFailure); - } - } - TableMetadataMaintenanceJobStatus::Disabled => { - actions.push(TableMaintenanceRecommendedAction::EnableBackgroundMaintenance); - } - TableMetadataMaintenanceJobStatus::Paused => { - actions.push(TableMaintenanceRecommendedAction::ResumeMaintenanceWorker); - } - } - actions -} - -fn push_unique_maintenance_action( - actions: &mut Vec, - action: TableMaintenanceRecommendedAction, -) { - if !actions.contains(&action) { - actions.push(action); - } -} - -fn table_maintenance_report_order_timestamp(report: &TableMetadataMaintenanceReport) -> String { - report - .job - .finished_at - .clone() - .or_else(|| report.job.heartbeat_at.clone()) - .or_else(|| report.job.started_at.clone()) - .or_else(|| report.job.scheduled_at.clone()) - .unwrap_or_default() -} - -fn table_maintenance_scheduler_job_summary(report: &TableMetadataMaintenanceReport) -> TableMaintenanceSchedulerJobSummary { - TableMaintenanceSchedulerJobSummary { - job_id: report.job.job_id.clone(), - operation: report.job.operation.clone(), - status: report.job.status.clone(), - scheduler_id: report.job.scheduler_id.clone(), - scheduled_at: report.job.scheduled_at.clone(), - worker_id: report.job.worker_id.clone(), - attempt: report.job.attempt, - started_at: report.job.started_at.clone(), - finished_at: report.job.finished_at.clone(), - heartbeat_at: report.job.heartbeat_at.clone(), - next_retry_after: report.job.next_retry_after.clone(), - recommended_actions: report.job.recommended_actions.clone(), - audit_events: report.audit_events.clone(), - } -} - -fn table_maintenance_scheduler_quarantine_boundary( - config: &TableMaintenanceConfig, - reports: &[TableMetadataMaintenanceReport], -) -> TableMaintenanceSchedulerQuarantineBoundary { - let source = reports - .iter() - .find(|report| report.job.quarantine_enabled && report.job.quarantined_object_count > 0); - TableMaintenanceSchedulerQuarantineBoundary { - enabled: config.quarantine_enabled, - active: source.is_some(), - retention_seconds: source.map_or(config.quarantine_retention_seconds, |report| report.job.quarantine_retention_seconds), - quarantined_object_count: source.map_or(0, |report| report.job.quarantined_object_count), - source_job_id: source.map(|report| report.job.job_id.clone()), - } -} - -fn refresh_table_maintenance_report_recommended_actions(report: &mut TableMetadataMaintenanceReport) { - report.job.recommended_actions = table_maintenance_recommended_actions(&report.job); -} - -fn table_maintenance_report_with_recommended_actions( - mut report: TableMetadataMaintenanceReport, -) -> TableMetadataMaintenanceReport { - refresh_table_maintenance_report_recommended_actions(&mut report); - report -} - -fn table_maintenance_scheduler_lease_is_active( - job: &TableMetadataMaintenanceJob, - scheduler_lease_timeout_seconds: u64, - now: OffsetDateTime, -) -> bool { - let Some(scheduled_at) = job.scheduled_at.as_deref().and_then(parse_maintenance_timestamp) else { - return false; - }; - let timeout_seconds = i64::try_from(scheduler_lease_timeout_seconds).unwrap_or(i64::MAX); - scheduled_at.saturating_add(Duration::seconds(timeout_seconds)) > now -} - -fn table_maintenance_job_lease_is_active( - job: &TableMetadataMaintenanceJob, - worker_lease_timeout_seconds: u64, - now: OffsetDateTime, -) -> bool { - let Some(heartbeat_at) = job.heartbeat_at.as_deref().and_then(parse_maintenance_timestamp) else { - return false; - }; - let timeout_seconds = i64::try_from(worker_lease_timeout_seconds).unwrap_or(i64::MAX); - heartbeat_at.saturating_add(Duration::seconds(timeout_seconds)) > now -} - -fn table_maintenance_job_retry_is_pending(job: &TableMetadataMaintenanceJob, now: OffsetDateTime) -> bool { - if !matches!(job.status, TableMetadataMaintenanceJobStatus::Failed) { - return false; - } - let Some(next_retry_after) = job.next_retry_after.as_deref().and_then(parse_maintenance_timestamp) else { - return false; - }; - next_retry_after > now -} - -fn validate_catalog_entry_version(kind: &str, version: u16) -> TableCatalogStoreResult<()> { - if version != TABLE_CATALOG_ENTRY_VERSION { - return Err(TableCatalogStoreError::Invalid(format!("unsupported {kind} entry version"))); - } - Ok(()) -} - -fn validate_table_maintenance_config_version(version: u16) -> TableCatalogStoreResult<()> { - if version != TABLE_MAINTENANCE_CONFIG_VERSION { - return Err(TableCatalogStoreError::Invalid( - "unsupported table maintenance config entry version".to_string(), - )); - } - Ok(()) -} - -fn validate_table_maintenance_config(config: &TableMaintenanceConfig) -> TableCatalogStoreResult<()> { - validate_table_maintenance_config_version(config.version)?; - if config.worker_lease_timeout_seconds == 0 { - return Err(TableCatalogStoreError::Invalid( - "worker-lease-timeout-seconds must be greater than zero".to_string(), - )); - } - if config.worker_lease_timeout_seconds > TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_MAX_SECONDS { - return Err(TableCatalogStoreError::Invalid(format!( - "worker-lease-timeout-seconds cannot exceed {TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_MAX_SECONDS}" - ))); - } - if config.max_retry_attempts > 10 { - return Err(TableCatalogStoreError::Invalid("max-retry-attempts cannot exceed 10".to_string())); - } - if config.max_retry_attempts > 0 && config.retry_initial_backoff_seconds == 0 { - return Err(TableCatalogStoreError::Invalid( - "retry-initial-backoff-seconds must be greater than zero when retry is enabled".to_string(), - )); - } - if config.max_retry_attempts > 0 && config.retry_initial_backoff_seconds > TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS { - return Err(TableCatalogStoreError::Invalid(format!( - "retry-initial-backoff-seconds cannot exceed {TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS}" - ))); - } - if config.max_retry_attempts > 0 && config.retry_max_backoff_seconds > TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS { - return Err(TableCatalogStoreError::Invalid(format!( - "retry-max-backoff-seconds cannot exceed {TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS}" - ))); - } - if config.max_retry_attempts > 0 && config.retry_max_backoff_seconds < config.retry_initial_backoff_seconds { - return Err(TableCatalogStoreError::Invalid( - "retry-max-backoff-seconds must be greater than or equal to retry-initial-backoff-seconds".to_string(), - )); - } - if config.quarantine_enabled && config.quarantine_retention_seconds == 0 { - return Err(TableCatalogStoreError::Invalid( - "quarantine-retention-seconds must be greater than zero when quarantine is enabled".to_string(), - )); - } - Ok(()) -} - -fn apply_maintenance_retry_after(job: &mut TableMetadataMaintenanceJob, config: &TableMaintenanceConfig, now: OffsetDateTime) { - if config.max_retry_attempts == 0 || job.attempt >= config.max_retry_attempts { - job.next_retry_after = None; - return; - } - let attempt_index = u32::from(job.attempt.saturating_sub(1)); - let multiplier = 1_u64.checked_shl(attempt_index).unwrap_or(u64::MAX); - let delay_seconds = config - .retry_initial_backoff_seconds - .saturating_mul(multiplier) - .min(config.retry_max_backoff_seconds); - let delay_seconds = i64::try_from(delay_seconds).unwrap_or(i64::MAX); - job.next_retry_after = Some(maintenance_timestamp(now.saturating_add(Duration::seconds(delay_seconds)))); -} - -fn validate_table_snapshot_expiration_config(config: &TableSnapshotExpirationConfig) -> TableCatalogStoreResult<()> { - if config.min_snapshots_to_keep == 0 { - return Err(TableCatalogStoreError::Invalid( - "min-snapshots-to-keep must be greater than zero".to_string(), - )); - } - if config.max_snapshot_age_ms < 0 { - return Err(TableCatalogStoreError::Invalid("max-snapshot-age-ms cannot be negative".to_string())); - } - Ok(()) -} - -fn validate_table_compaction_planning_config(config: &TableCompactionPlanningConfig) -> TableCatalogStoreResult<()> { - if config.target_file_size_bytes == 0 { - return Err(TableCatalogStoreError::Invalid( - "target-file-size-bytes must be greater than zero".to_string(), - )); - } - if config.small_file_threshold_bytes == 0 { - return Err(TableCatalogStoreError::Invalid( - "small-file-threshold-bytes must be greater than zero".to_string(), - )); - } - if config.small_file_threshold_bytes > config.target_file_size_bytes { - return Err(TableCatalogStoreError::Invalid( - "small-file-threshold-bytes cannot exceed target-file-size-bytes".to_string(), - )); - } - if config.min_input_files < 2 { - return Err(TableCatalogStoreError::Invalid("min-input-files must be at least two".to_string())); - } - if config.max_rewrite_bytes_per_job < config.target_file_size_bytes { - return Err(TableCatalogStoreError::Invalid( - "max-rewrite-bytes-per-job must be at least target-file-size-bytes".to_string(), - )); - } - Ok(()) -} - -fn commit_log_matches_request(commit_log: &CommitLogEntry, request: &TableCommitRequest, table_id: &str) -> bool { - commit_log.version == TABLE_CATALOG_ENTRY_VERSION - && commit_log.commit_id == request.commit_id - && commit_log.idempotency_key == request.idempotency_key - && commit_log.table_id == table_id - && commit_log.operation == request.operation - && commit_log.expected_version_token == request.expected_version_token - && commit_log.previous_metadata_location == request.expected_metadata_location - && commit_log.new_metadata_location == request.new_metadata_location - && commit_log.requirements == request.requirements - && commit_log.writer == request.writer -} - -fn table_matches_committed_log(table: &TableEntry, commit_log: &CommitLogEntry) -> bool { - table.table_id == commit_log.table_id - && table.metadata_location == commit_log.new_metadata_location - && table.version_token == commit_log.new_version_token -} - -fn table_matches_staged_base(table: &TableEntry, commit_log: &CommitLogEntry) -> bool { - table.table_id == commit_log.table_id - && table.metadata_location == commit_log.previous_metadata_location - && table.version_token == commit_log.expected_version_token -} - -fn table_catalog_recovery_summary( - metadata_status: &TableMetadataPointerStatus, - commit_recovery: &TableCommitRecoveryReport, -) -> (TableCatalogRecoveryStatus, Vec) { - let mut actions = Vec::new(); - let metadata_status = match metadata_status { - TableMetadataPointerStatus::Valid => None, - TableMetadataPointerStatus::MissingObject => { - actions.push(TableCatalogRecoveryAction::RestoreCurrentMetadataObject); - Some(TableCatalogRecoveryStatus::ReadOnlyRecommended) - } - TableMetadataPointerStatus::InvalidJson => { - actions.push(TableCatalogRecoveryAction::FixCurrentMetadataJson); - Some(TableCatalogRecoveryStatus::ReadOnlyRecommended) - } - TableMetadataPointerStatus::InvalidLocation => { - actions.push(TableCatalogRecoveryAction::MoveCurrentMetadataInsideTable); - Some(TableCatalogRecoveryStatus::ReadOnlyRecommended) - } - }; - - if commit_recovery.manual_review_count > 0 { - actions.push(TableCatalogRecoveryAction::ReviewCommitLog); - return (metadata_status.unwrap_or(TableCatalogRecoveryStatus::ManualReviewRequired), actions); - } - if commit_recovery.finalization_required_count > 0 || commit_recovery.idempotency_repair_required_count > 0 { - actions.push(TableCatalogRecoveryAction::RunCommitRecovery); - return (metadata_status.unwrap_or(TableCatalogRecoveryStatus::Recoverable), actions); - } - if commit_recovery.staged_before_table_update_count > 0 { - actions.push(TableCatalogRecoveryAction::RetryCommit); - return (metadata_status.unwrap_or(TableCatalogRecoveryStatus::Recoverable), actions); - } - - (metadata_status.unwrap_or(TableCatalogRecoveryStatus::Healthy), actions) -} - -fn commit_logs_share_recovery_payload(left: &CommitLogEntry, right: &CommitLogEntry) -> bool { - left.version == right.version - && left.commit_id == right.commit_id - && left.idempotency_key == right.idempotency_key - && left.table_id == right.table_id - && left.operation == right.operation - && left.expected_version_token == right.expected_version_token - && left.new_version_token == right.new_version_token - && left.previous_metadata_location == right.previous_metadata_location - && left.new_metadata_location == right.new_metadata_location - && left.requirements == right.requirements - && left.writer == right.writer -} - -fn commit_idempotency_index_status( - commit_log: &CommitLogEntry, - idempotency_commit: Option<&CommitLogEntry>, -) -> TableCommitIdempotencyIndexStatus { - match (commit_log.idempotency_key.as_ref(), idempotency_commit) { - (None, _) => TableCommitIdempotencyIndexStatus::NotRequired, - (Some(_), None) => TableCommitIdempotencyIndexStatus::Missing, - (Some(_), Some(indexed)) if indexed == commit_log => TableCommitIdempotencyIndexStatus::Matches, - (Some(_), Some(indexed)) if commit_logs_share_recovery_payload(indexed, commit_log) => { - TableCommitIdempotencyIndexStatus::Stale - } - (Some(_), Some(_)) => TableCommitIdempotencyIndexStatus::Conflicting, - } -} - -fn table_commit_recovery_entry( - table: &TableEntry, - commit_log: &CommitLogEntry, - idempotency_commit: Option<&CommitLogEntry>, -) -> TableCommitRecoveryEntry { - let idempotency_index_status = commit_idempotency_index_status(commit_log, idempotency_commit); - let idempotency_index_present = matches!( - idempotency_index_status, - TableCommitIdempotencyIndexStatus::Matches - | TableCommitIdempotencyIndexStatus::Stale - | TableCommitIdempotencyIndexStatus::Conflicting - ); - let idempotency_index_repair_required = matches!( - idempotency_index_status, - TableCommitIdempotencyIndexStatus::Missing | TableCommitIdempotencyIndexStatus::Stale - ); - - let (recovery_state, reason) = if matches!(idempotency_index_status, TableCommitIdempotencyIndexStatus::Conflicting) { - ( - TableCommitRecoveryState::ManualReview, - "idempotency index points at a different commit payload".to_string(), - ) - } else if table_matches_committed_log(table, commit_log) { - if matches!(commit_log.status, CommitLogStatus::Committed) { - if idempotency_index_repair_required { - ( - TableCommitRecoveryState::IdempotencyIndexRepairRequired, - "committed table pointer is durable but idempotency index needs repair".to_string(), - ) - } else { - ( - TableCommitRecoveryState::Committed, - "commit log and current table pointer agree".to_string(), - ) - } - } else { - ( - TableCommitRecoveryState::FinalizationRequired, - "current table pointer already advanced but commit log is not finalized".to_string(), - ) - } - } else if matches!(commit_log.status, CommitLogStatus::Committed) { - if idempotency_index_repair_required { - ( - TableCommitRecoveryState::IdempotencyIndexRepairRequired, - "historical committed log needs idempotency index repair".to_string(), - ) - } else { - ( - TableCommitRecoveryState::Committed, - "commit is finalized and may be older than the current table pointer".to_string(), - ) - } - } else if table_matches_staged_base(table, commit_log) { - ( - TableCommitRecoveryState::StagedBeforeTableUpdate, - "staged commit exists but table pointer has not advanced".to_string(), - ) - } else { - ( - TableCommitRecoveryState::ManualReview, - "staged commit no longer matches the current table pointer or its expected base".to_string(), - ) - }; - - TableCommitRecoveryEntry { - commit_id: commit_log.commit_id.clone(), - idempotency_key: commit_log.idempotency_key.clone(), - operation: commit_log.operation.clone(), - status: commit_log.status.clone(), - recovery_state, - previous_metadata_location: commit_log.previous_metadata_location.clone(), - new_metadata_location: commit_log.new_metadata_location.clone(), - expected_version_token: commit_log.expected_version_token.clone(), - new_version_token: commit_log.new_version_token.clone(), - idempotency_index_present, - idempotency_index_status, - reason, - } -} - -fn record_table_commit_attempt(operation: &str) { - counter!("rustfs_table_catalog_commit_attempts_total", "operation" => operation.to_string()).increment(1); -} - -fn table_catalog_store_result_label(result: &TableCatalogStoreResult) -> &'static str { - match result { - Ok(_) => "success", - Err(TableCatalogStoreError::Conflict(_)) => "conflict", - Err(TableCatalogStoreError::Invalid(_)) => "invalid", - Err(TableCatalogStoreError::NotFound(_)) => "not_found", - Err(TableCatalogStoreError::Internal(_)) => "failure", - } -} - -fn duration_millis_u64(duration: StdDuration) -> u64 { - u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) -} - -fn record_table_commit_cas_result(operation: &str, started: Instant, result: &TableCatalogStoreResult<()>) { - let elapsed = started.elapsed(); - let result_label = table_catalog_store_result_label(result); - counter!( - "rustfs_table_catalog_commit_cas_results_total", - "operation" => operation.to_string(), - "result" => result_label.to_string() - ) - .increment(1); - histogram!( - "rustfs_table_catalog_commit_cas_duration_seconds", - "operation" => operation.to_string(), - "result" => result_label.to_string() - ) - .record(elapsed.as_secs_f64()); -} - -fn record_table_commit_result( - table_bucket: &str, - namespace: &str, - table: &str, - commit_id: &str, - operation: &str, - started: Instant, - result: &TableCatalogStoreResult, -) { - let elapsed = started.elapsed(); - let result_label = table_catalog_store_result_label(result); - counter!( - "rustfs_table_catalog_commit_results_total", - "operation" => operation.to_string(), - "result" => result_label.to_string() - ) - .increment(1); - if matches!(result, Err(TableCatalogStoreError::Conflict(_))) { - counter!("rustfs_table_catalog_commit_conflicts_total", "operation" => operation.to_string()).increment(1); - } - histogram!( - "rustfs_table_catalog_commit_duration_seconds", - "operation" => operation.to_string(), - "result" => result_label.to_string() - ) - .record(elapsed.as_secs_f64()); - - match result { - Ok(commit) if elapsed >= TABLE_COMMIT_SLOW_LOG_THRESHOLD => { - tracing::warn!( - table_bucket, - namespace, - table, - commit_id, - operation, - generation = commit.table.generation, - duration_ms = duration_millis_u64(elapsed), - "slow table catalog commit" - ); - } - Ok(commit) => { - tracing::debug!( - table_bucket, - namespace, - table, - commit_id, - operation, - generation = commit.table.generation, - duration_ms = duration_millis_u64(elapsed), - "table catalog commit completed" - ); - } - Err(error) => { - tracing::warn!( - table_bucket, - namespace, - table, - commit_id, - operation, - result = result_label, - duration_ms = duration_millis_u64(elapsed), - error = %error, - "table catalog commit did not complete" - ); - } - } -} - -fn table_commit_result( - table_bucket: &str, - namespace: &str, - table: &str, - commit_id: &str, - operation: &str, - started: Instant, - result: TableCatalogStoreResult, -) -> TableCatalogStoreResult { - record_table_commit_result(table_bucket, namespace, table, commit_id, operation, started, &result); - result -} - -fn http_preconditions_for_catalog_put(precondition: TableCatalogPutPrecondition) -> Option { - match precondition { - TableCatalogPutPrecondition::Any => None, - TableCatalogPutPrecondition::IfAbsent => Some(HTTPPreconditions { - if_none_match: Some("*".to_string()), - ..Default::default() - }), - TableCatalogPutPrecondition::IfMatch(etag) => Some(HTTPPreconditions { - if_match: Some(etag), - ..Default::default() - }), - } -} - -fn is_missing_storage_error(err: &StorageError) -> bool { - matches!( - err, - StorageError::ObjectNotFound(_, _) | StorageError::FileNotFound | StorageError::ConfigNotFound - ) -} - -fn storage_error_to_catalog(action: &str, err: StorageError) -> TableCatalogStoreError { - match err { - StorageError::ObjectNotFound(bucket, object) => TableCatalogStoreError::NotFound(format!("{action}: {bucket}/{object}")), - StorageError::BucketNotFound(bucket) => TableCatalogStoreError::NotFound(format!("{action}: bucket {bucket}")), - StorageError::PreconditionFailed => TableCatalogStoreError::Conflict(format!("{action}: precondition failed")), - other => TableCatalogStoreError::Internal(format!("{action}: {other}")), - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct NamespaceMarker { - pub version: u16, - pub namespace: String, -} - -impl NamespaceMarker { - pub fn new(namespace: &Namespace) -> Self { - Self { - version: TABLE_NAMESPACE_MARKER_VERSION, - namespace: namespace.public_name(), - } - } -} - -pub(crate) fn namespace_marker_json(namespace: &Namespace) -> Result, serde_json::Error> { - serde_json::to_vec(&NamespaceMarker::new(namespace)) -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct TableMarker { - pub version: u16, - pub namespace: String, - pub name: String, - pub metadata_location: Option, -} - -impl TableMarker { - pub fn new(namespace: &Namespace, table: &IdentifierSegment) -> Self { - Self { - version: TABLE_RESOURCE_MARKER_VERSION, - namespace: namespace.public_name(), - name: table.as_str().to_string(), - metadata_location: None, - } - } -} - -pub(crate) fn table_marker_json(namespace: &Namespace, table: &IdentifierSegment) -> Result, serde_json::Error> { - serde_json::to_vec(&TableMarker::new(namespace, table)) -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct TableMetadataPointer { - pub version: u16, - pub metadata_location: String, -} - -impl TableMetadataPointer { - pub fn new(metadata_location: String) -> Self { - Self { - version: TABLE_METADATA_POINTER_VERSION, - metadata_location, - } - } -} - -pub(crate) fn table_metadata_pointer_json(metadata_location: String) -> Result, serde_json::Error> { - serde_json::to_vec(&TableMetadataPointer::new(metadata_location)) -} - -pub(crate) fn parse_table_metadata_pointer(data: &[u8]) -> Result { - serde_json::from_slice(data) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct IdentifierSegment(String); - -impl IdentifierSegment { - pub const MAX_LEN: usize = 64; - - pub fn parse(value: impl Into) -> Result { - let value = value.into(); - validate_identifier_segment(&value)?; - Ok(Self(value)) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Namespace { - segments: Vec, -} - -impl Namespace { - pub const MAX_LEN: usize = 512; - - pub fn parse(value: &str) -> Result { - if value.is_empty() { - return Err(CatalogIdentifierError::Empty); - } - if value.len() > Self::MAX_LEN { - return Err(CatalogIdentifierError::NamespaceTooLong { max: Self::MAX_LEN }); - } - - let mut segments = Vec::new(); - for segment in value.split('.') { - segments.push(IdentifierSegment::parse(segment.to_string())?); - } - - Ok(Self { segments }) - } - - pub fn segments(&self) -> &[IdentifierSegment] { - &self.segments - } - - pub fn storage_id(&self) -> String { - self.segments - .iter() - .map(IdentifierSegment::as_str) - .collect::>() - .join("/") - } - - pub fn public_name(&self) -> String { - self.segments - .iter() - .map(IdentifierSegment::as_str) - .collect::>() - .join(".") - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TableIdentifier { - warehouse: IdentifierSegment, - namespace: Namespace, - name: IdentifierSegment, -} - -impl TableIdentifier { - pub fn new(warehouse: IdentifierSegment, namespace: Namespace, name: IdentifierSegment) -> Self { - Self { - warehouse, - namespace, - name, - } - } - - pub fn warehouse(&self) -> &IdentifierSegment { - &self.warehouse - } - - pub fn namespace(&self) -> &Namespace { - &self.namespace - } - - pub fn name(&self) -> &IdentifierSegment { - &self.name - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TablePathResolver { - reserved_prefix: &'static str, -} - -impl Default for TablePathResolver { - fn default() -> Self { - Self { - reserved_prefix: TABLE_RESERVED_PREFIX, - } - } -} - -impl TablePathResolver { - pub fn current_pointer_path(&self, table: &TableIdentifier) -> String { - format!("{}/{}", self.table_root(table), CURRENT_POINTER_FILE) - } - - pub fn metadata_dir_path(&self, table: &TableIdentifier) -> String { - format!("{}/{}", self.table_root(table), METADATA_DIR) - } - - pub fn metadata_file_path(&self, table: &TableIdentifier, metadata_file_name: &str) -> String { - format!("{}/{}", self.metadata_dir_path(table), metadata_file_name) - } - - fn table_root(&self, table: &TableIdentifier) -> String { - format!( - "{}/{}/{}/{}/{}/{}/{}", - self.reserved_prefix, - WAREHOUSE_ROOT, - table.warehouse().as_str(), - NAMESPACE_ROOT, - table.namespace().storage_id(), - TABLE_ROOT, - table.name().as_str() - ) - } -} - -pub fn is_reserved_table_object_key(object_key: &str) -> bool { - object_key == TABLE_RESERVED_PREFIX - || object_key - .strip_prefix(TABLE_RESERVED_PREFIX) - .is_some_and(|rest| rest.starts_with('/')) -} - -pub(crate) fn default_namespace_root_prefix() -> String { - format!( - "{}/{}/{}/{}/", - TABLE_RESERVED_PREFIX, WAREHOUSE_ROOT, DEFAULT_WAREHOUSE_ID, NAMESPACE_ROOT - ) -} - -pub(crate) fn default_namespace_marker_path(namespace: &Namespace) -> String { - format!("{}{}/{}", default_namespace_root_prefix(), namespace.storage_id(), NAMESPACE_MARKER_FILE) -} - -pub(crate) fn default_table_root_prefix(namespace: &Namespace) -> String { - format!("{}{}/{}/", default_namespace_root_prefix(), namespace.storage_id(), TABLE_ROOT) -} - -pub(crate) fn default_table_marker_path(namespace: &Namespace, table: &IdentifierSegment) -> String { - format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), TABLE_MARKER_FILE) -} - -pub(crate) fn default_table_metadata_dir_path(namespace: &Namespace, table: &IdentifierSegment) -> String { - format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), METADATA_DIR) -} - -pub(crate) fn default_table_data_dir_path(namespace: &Namespace, table: &IdentifierSegment) -> String { - format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), DATA_DIR) -} - -pub(crate) fn default_table_delete_dir_path(namespace: &Namespace, table: &IdentifierSegment) -> String { - format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), DELETE_DIR) -} - -pub(crate) fn default_view_root_prefix(namespace: &Namespace) -> String { - format!("{}{}/{}/", default_namespace_root_prefix(), namespace.storage_id(), VIEW_ROOT) -} - -pub(crate) fn default_view_metadata_dir_path(namespace: &Namespace, view: &IdentifierSegment) -> String { - format!("{}{}/{}", default_view_root_prefix(namespace), view.as_str(), METADATA_DIR) -} - -pub(crate) fn default_view_metadata_file_path( - namespace: &Namespace, - view: &IdentifierSegment, - metadata_file_name: &str, -) -> String { - format!("{}/{}", default_view_metadata_dir_path(namespace, view), metadata_file_name) -} - -pub(crate) fn default_table_metadata_file_path( - namespace: &Namespace, - table: &IdentifierSegment, - metadata_file_name: &str, -) -> String { - format!("{}/{}", default_table_metadata_dir_path(namespace, table), metadata_file_name) -} - -pub(crate) fn default_table_current_pointer_path(namespace: &Namespace, table: &IdentifierSegment) -> String { - format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), CURRENT_POINTER_FILE) -} - -pub(crate) fn default_table_lifecycle_path(namespace: &Namespace, table: &IdentifierSegment) -> String { - format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), LIFECYCLE_FILE) -} - -pub(crate) fn namespace_name_from_marker_path(object_key: &str) -> Option { - let prefix = default_namespace_root_prefix(); - let suffix = format!("/{NAMESPACE_MARKER_FILE}"); - - object_key - .strip_prefix(prefix.as_str()) - .and_then(|value| value.strip_suffix(suffix.as_str())) - .filter(|value| !value.is_empty()) - .map(|value| value.replace('/', ".")) -} - -pub(crate) fn table_name_from_marker_path(namespace: &Namespace, object_key: &str) -> Option { - let prefix = default_table_root_prefix(namespace); - let suffix = format!("/{TABLE_MARKER_FILE}"); - - object_key - .strip_prefix(prefix.as_str()) - .and_then(|value| value.strip_suffix(suffix.as_str())) - .filter(|value| !value.is_empty() && !value.contains('/')) - .map(ToString::to_string) -} - -pub(crate) fn metadata_location_from_metadata_file_path( - namespace: &Namespace, - table: &IdentifierSegment, - object_key: &str, -) -> Option { - let prefix = format!("{}/", default_table_metadata_dir_path(namespace, table)); - - object_key - .strip_prefix(prefix.as_str()) - .filter(|value| is_valid_table_metadata_file_name(value)) - .map(|_| object_key.to_string()) -} - -pub(crate) fn is_valid_table_metadata_location( - namespace: &Namespace, - table: &IdentifierSegment, - metadata_location: &str, -) -> bool { - if metadata_location.is_empty() { - return false; - } - - let metadata_prefix = format!("{}/", default_table_metadata_dir_path(namespace, table)); - metadata_location - .strip_prefix(&metadata_prefix) - .is_some_and(is_valid_table_metadata_file_name) -} - -pub(crate) fn is_valid_view_metadata_location(namespace: &Namespace, view: &IdentifierSegment, metadata_location: &str) -> bool { - if metadata_location.is_empty() { - return false; - } - - let metadata_prefix = format!("{}/", default_view_metadata_dir_path(namespace, view)); - metadata_location - .strip_prefix(&metadata_prefix) - .is_some_and(is_valid_table_metadata_file_name) -} - -pub(crate) fn is_valid_table_metadata_file_name(metadata_file_name: &str) -> bool { - if metadata_file_name.is_empty() - || metadata_file_name.len() > TABLE_METADATA_FILE_NAME_MAX_LEN - || !metadata_file_name.ends_with(".json") - || metadata_file_name.contains("..") - || metadata_file_name.contains('%') - || metadata_file_name.contains('/') - || metadata_file_name.contains('\\') - || metadata_file_name.bytes().any(|byte| byte.is_ascii_control()) - { - return false; - } - - let bytes = metadata_file_name.as_bytes(); - if !is_lower_ascii_alnum(bytes[0]) || !is_lower_ascii_alnum(bytes[bytes.len() - 1]) { - return false; - } - - bytes - .iter() - .all(|byte| is_lower_ascii_alnum(*byte) || matches!(*byte, b'.' | b'_' | b'-')) -} - -pub fn validate_object_mutation(table_bucket_enabled: bool, object_key: &str) -> Result<(), TableObjectMutationError> { - if table_bucket_enabled && is_reserved_table_object_key(object_key) { - return Err(TableObjectMutationError::ReservedCatalogObject); - } - - Ok(()) -} - -pub(crate) async fn validate_bucket_object_mutation(bucket: &str, object_key: &str) -> Result<(), TableObjectMutationError> { - if !is_reserved_table_object_key(object_key) { - return Ok(()); - } - - let table_bucket_enabled = get_bucket_metadata(bucket) - .await - .map(|metadata| metadata.table_bucket_enabled()) - .unwrap_or(true); - - validate_object_mutation(table_bucket_enabled, object_key) -} - -fn validate_identifier_segment(value: &str) -> Result<(), CatalogIdentifierError> { - if value.is_empty() { - return Err(CatalogIdentifierError::Empty); - } - - if value.len() > IdentifierSegment::MAX_LEN { - return Err(CatalogIdentifierError::TooLong { - max: IdentifierSegment::MAX_LEN, - }); - } - - if matches!(value, "." | "..") || value.contains('%') || value.contains('/') || value.contains('\\') { - return Err(CatalogIdentifierError::Ambiguous); - } - - let bytes = value.as_bytes(); - if !is_lower_ascii_alnum(bytes[0]) || !is_lower_ascii_alnum(bytes[bytes.len() - 1]) { - return Err(CatalogIdentifierError::InvalidBoundary); - } - - if bytes - .iter() - .any(|byte| !is_lower_ascii_alnum(*byte) && !matches!(*byte, b'_' | b'-')) - { - return Err(CatalogIdentifierError::InvalidCharacter); - } - - Ok(()) -} - -fn is_lower_ascii_alnum(value: u8) -> bool { - value.is_ascii_lowercase() || value.is_ascii_digit() -} - -#[cfg(test)] -mod tests { - use super::*; - use datafusion::{ - arrow::{ - array::{Array, Int32Array, Int64Array}, - datatypes::{DataType, Field, Schema, SchemaRef}, - record_batch::RecordBatch, - }, - parquet::arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder}, - }; - use std::assert_matches; - use std::sync::Arc; - - #[test] - fn reserved_table_object_key_matches_exact_prefix_and_children_only() { - assert!(is_reserved_table_object_key(".rustfs-table")); - assert!(is_reserved_table_object_key(".rustfs-table/")); - assert!(is_reserved_table_object_key(".rustfs-table/metadata/current.json")); - - assert!(!is_reserved_table_object_key("")); - assert!(!is_reserved_table_object_key(".rustfs-table-other")); - assert!(!is_reserved_table_object_key("prefix/.rustfs-table/object")); - assert!(!is_reserved_table_object_key("user/.rustfs-table")); - } - - #[test] - fn object_mutation_guard_only_blocks_reserved_prefix_for_table_buckets() { - assert!(validate_object_mutation(false, ".rustfs-table/current.json").is_ok()); - assert_eq!( - validate_object_mutation(true, ".rustfs-table/current.json").unwrap_err(), - TableObjectMutationError::ReservedCatalogObject - ); - assert!(validate_object_mutation(true, ".rustfs-table-other/current.json").is_ok()); - } - - #[tokio::test] - async fn bucket_object_mutation_guard_fails_closed_for_reserved_prefix_when_bucket_metadata_is_unavailable() { - assert_eq!( - validate_bucket_object_mutation("missing-bucket", ".rustfs-table/current.json") - .await - .unwrap_err(), - TableObjectMutationError::ReservedCatalogObject - ); - assert!( - validate_bucket_object_mutation("missing-bucket", "ordinary/current.json") - .await - .is_ok() - ); - } - - #[test] - fn table_bucket_marker_json_uses_stable_catalog_defaults() { - let marker = serde_json::to_value(TableBucketMarker::default()).unwrap(); - - assert_eq!(marker["version"], TABLE_BUCKET_CONFIG_VERSION); - assert_eq!(marker["catalog_type"], TABLE_BUCKET_CATALOG_TYPE); - assert_eq!(marker["reserved_prefix"], TABLE_RESERVED_PREFIX); - assert!(!table_bucket_marker_json().unwrap().is_empty()); - } - - #[test] - fn catalog_entry_structures_serialize_stable_fields() { - use std::collections::BTreeMap; - - let bucket = TableBucketEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "analytics".to_string(), - catalog_type: TABLE_BUCKET_CATALOG_TYPE.to_string(), - warehouse_root: "s3://analytics/".to_string(), - state: TableCatalogEntryState::Active, - properties: BTreeMap::from([("owner".to_string(), "platform".to_string())]), - created_at: Some("2026-05-23T00:00:00Z".to_string()), - updated_at: Some("2026-05-23T00:00:00Z".to_string()), - }; - let namespace = NamespaceEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "analytics".to_string(), - namespace: "sales".to_string(), - namespace_id: "sales".to_string(), - state: TableCatalogEntryState::Active, - properties: BTreeMap::from([("purpose".to_string(), "orders".to_string())]), - created_at: Some("2026-05-23T00:00:00Z".to_string()), - updated_at: Some("2026-05-23T00:00:00Z".to_string()), - }; - let table = TableEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "analytics".to_string(), - namespace: "sales".to_string(), - table: "orders".to_string(), - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: "s3://analytics/tables/table-id".to_string(), - metadata_location: "s3://analytics/tables/table-id/metadata/v1.metadata.json".to_string(), - version_token: "token-v1".to_string(), - generation: 1, - state: TableCatalogEntryState::Active, - properties: BTreeMap::from([("write.format.default".to_string(), "parquet".to_string())]), - created_at: Some("2026-05-23T00:00:00Z".to_string()), - updated_at: Some("2026-05-23T00:00:00Z".to_string()), - }; - let commit = CommitLogEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - commit_id: "commit-id".to_string(), - idempotency_key: Some("client-request-id".to_string()), - table_id: "table-id".to_string(), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - new_version_token: "token-v2".to_string(), - previous_metadata_location: "s3://analytics/tables/table-id/metadata/v1.metadata.json".to_string(), - new_metadata_location: "s3://analytics/tables/table-id/metadata/v2.metadata.json".to_string(), - requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], - status: CommitLogStatus::Committed, - writer: Some("pyiceberg/test".to_string()), - created_at: Some("2026-05-23T00:01:00Z".to_string()), - updated_at: Some("2026-05-23T00:01:00Z".to_string()), - }; - - let bucket_json = serde_json::to_value(&bucket).unwrap(); - let namespace_json = serde_json::to_value(&namespace).unwrap(); - let table_json = serde_json::to_value(&table).unwrap(); - let commit_json = serde_json::to_value(&commit).unwrap(); - - assert_eq!(bucket_json["state"], "ACTIVE"); - assert_eq!(bucket_json["properties"]["owner"], "platform"); - assert_eq!(namespace_json["namespace_id"], "sales"); - assert_eq!(table_json["version_token"], "token-v1"); - assert_eq!(table_json["generation"], 1); - assert_eq!(table_json["state"], "ACTIVE"); - assert_eq!(commit_json["status"], "COMMITTED"); - assert_eq!(commit_json["requirements"][0]["type"], "assert-table-uuid"); - } - - #[test] - fn catalog_entry_deserialization_rejects_unknown_fields() { - use std::collections::BTreeMap; - - let table = TableEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: "analytics".to_string(), - namespace: "sales".to_string(), - table: "orders".to_string(), - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: "s3://analytics/tables/table-id".to_string(), - metadata_location: "s3://analytics/tables/table-id/metadata/v1.metadata.json".to_string(), - version_token: "token-v1".to_string(), - generation: 1, - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }; - let mut value = serde_json::to_value(table).unwrap(); - value - .as_object_mut() - .unwrap() - .insert("unexpected".to_string(), serde_json::json!(true)); - - assert!(serde_json::from_value::(value).is_err()); - } - - #[test] - #[serial_test::serial] - fn table_catalog_backing_mode_defaults_to_object() { - temp_env::with_var_unset(ENV_TABLE_CATALOG_BACKING, || { - assert_eq!(TableCatalogBackingMode::from_env().unwrap(), TableCatalogBackingMode::ObjectBacked); - }); - } - - #[test] - #[serial_test::serial] - fn table_catalog_backing_mode_accepts_durable_strong_value() { - temp_env::with_var(ENV_TABLE_CATALOG_BACKING, Some(TABLE_CATALOG_BACKING_DURABLE_STRONG), || { - assert_eq!(TableCatalogBackingMode::from_env().unwrap(), TableCatalogBackingMode::DurableStrong); - }); - } - - #[test] - #[serial_test::serial] - fn table_catalog_backing_mode_rejects_unknown_value() { - temp_env::with_var(ENV_TABLE_CATALOG_BACKING, Some("memory"), || { - assert!(matches!( - TableCatalogBackingMode::from_env().unwrap_err(), - TableCatalogStoreError::Invalid(_) - )); - }); - } - - struct NoopTableCatalogStore; - - #[async_trait::async_trait] - impl TableCatalogStore for NoopTableCatalogStore { - async fn get_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn put_table_bucket(&self, _entry: TableBucketEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn create_namespace(&self, _entry: NamespaceEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn list_namespaces(&self, _table_bucket: &str) -> TableCatalogStoreResult> { - Ok(Vec::new()) - } - - async fn get_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn drop_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn create_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn register_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn list_tables(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { - Ok(Vec::new()) - } - - async fn load_table( - &self, - _table_bucket: &str, - _namespace: &str, - _table: &str, - ) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { - let table = TableEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: request.table_bucket, - namespace: request.namespace, - table: request.table, - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: "s3://analytics/tables/table-id".to_string(), - metadata_location: request.new_metadata_location.clone(), - version_token: "token-v2".to_string(), - generation: 2, - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }; - let commit_log = CommitLogEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - commit_id: request.commit_id, - idempotency_key: request.idempotency_key, - table_id: table.table_id.clone(), - operation: request.operation, - expected_version_token: request.expected_version_token, - new_version_token: table.version_token.clone(), - previous_metadata_location: request.expected_metadata_location, - new_metadata_location: table.metadata_location.clone(), - requirements: request.requirements, - status: CommitLogStatus::Committed, - writer: request.writer, - created_at: None, - updated_at: None, - }; - - Ok(TableCommitResult { table, commit_log }) - } - - async fn drop_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn create_view(&self, _entry: ViewEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn list_views(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { - Ok(Vec::new()) - } - - async fn load_view( - &self, - _table_bucket: &str, - _namespace: &str, - _view: &str, - ) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { - Ok(ViewCommitResult { - view: ViewEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: request.table_bucket, - namespace: request.namespace, - view: request.view, - view_id: "view-id".to_string(), - view_uuid: "view-uuid".to_string(), - format: "ICEBERG_VIEW".to_string(), - format_version: 1, - warehouse_location: "s3://analytics/views/view-id".to_string(), - metadata_location: request.new_metadata_location, - version_token: "token-v2".to_string(), - generation: 2, - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }, - }) - } - - async fn drop_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn get_commit_by_id( - &self, - _table_bucket: &str, - _table_id: &str, - _commit_id: &str, - ) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn get_commit_by_idempotency_key( - &self, - _table_bucket: &str, - _table_id: &str, - _idempotency_key: &str, - ) -> TableCatalogStoreResult> { - Ok(None) - } - } - - #[tokio::test] - async fn table_catalog_store_trait_covers_entry_read_write_shapes() { - let store: &dyn TableCatalogStore = &NoopTableCatalogStore; - - assert!(store.get_table_bucket("analytics").await.unwrap().is_none()); - assert!(store.list_namespaces("analytics").await.unwrap().is_empty()); - assert!( - store - .get_commit_by_id("analytics", "table-id", "commit-id") - .await - .unwrap() - .is_none() - ); - assert!( - store - .get_commit_by_idempotency_key("analytics", "table-id", "client-request-id") - .await - .unwrap() - .is_none() - ); - } - - #[tokio::test] - async fn table_catalog_store_trait_has_atomic_commit_shape() { - let store: &dyn TableCatalogStore = &NoopTableCatalogStore; - let request = TableCommitRequest { - table_bucket: "analytics".to_string(), - namespace: "sales".to_string(), - table: "orders".to_string(), - commit_id: "commit-id".to_string(), - idempotency_key: Some("client-request-id".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: "s3://analytics/tables/table-id/metadata/v1.metadata.json".to_string(), - new_metadata_location: "s3://analytics/tables/table-id/metadata/v2.metadata.json".to_string(), - requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], - writer: Some("pyiceberg/test".to_string()), - }; - - let result = store.commit_table(request).await.unwrap(); - - assert_eq!(result.table.version_token, "token-v2"); - assert_eq!(result.table.generation, 2); - assert_eq!(result.commit_log.status, CommitLogStatus::Committed); - } - - #[test] - fn catalog_object_entry_paths_use_internal_root_and_hashed_untrusted_ids() { - let paths = TableCatalogObjectPaths::default(); - let bucket = "analytics"; - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - let table = IdentifierSegment::parse("events").unwrap(); - let bucket_root = format!("s3tables/catalog/table-buckets/{}/", table_catalog_path_hash(bucket)); - - assert_eq!(paths.table_bucket_entry_path(bucket), format!("{bucket_root}table-bucket.json")); - assert_eq!( - paths.namespace_entry_path(bucket, &namespace), - format!("{bucket_root}namespaces/analytics/daily_events/namespace-entry.json") - ); - assert_eq!( - paths.table_entry_path(bucket, &namespace, &table), - format!("{bucket_root}namespaces/analytics/daily_events/tables/events/table-entry.json") - ); - assert_eq!( - paths.view_entry_path(bucket, &namespace, &table), - format!("{bucket_root}namespaces/analytics/daily_events/views/events/view-entry.json") - ); - - let commit_path = paths.commit_log_entry_path("table/../bucket", "table/../id", "commit/%2f\nid"); - let idempotency_path = paths.commit_idempotency_entry_path("table/../bucket", "table/../id", "client/%2f\nrequest"); - let warehouse_index_path = paths.warehouse_index_entry_path("table/../bucket", "tables/table/../id/data\nprefix/"); - let warehouse_index_state_path = paths.warehouse_index_state_path("table/../bucket"); - let maintenance_config_path = paths.table_maintenance_config_path("table/../bucket", &namespace, &table, "table/../id"); - let maintenance_job_path = - paths.table_maintenance_job_path("table/../bucket", &namespace, &table, "table/../id", "job/%2f\nid"); - - for path in [ - commit_path, - idempotency_path, - warehouse_index_path, - warehouse_index_state_path, - maintenance_config_path, - maintenance_job_path, - ] { - assert!(path.starts_with("s3tables/catalog/table-buckets/")); - assert!(path.ends_with(".json")); - assert!(!path.contains("..")); - assert!(!path.contains('%')); - assert!(!path.contains('\n')); - assert!(!path.contains("table/../bucket")); - assert!(!path.contains("table/../id")); - assert!(!path.contains("client/%2f")); - } - } - - #[derive(Clone, Default)] - struct TestCatalogObjectBackend { - state: Arc>, - locks: TestCatalogObjectLocks, - } - - type TestCatalogObjectLockKey = (String, String); - type TestCatalogObjectLock = Arc>; - type TestCatalogObjectLocks = Arc>>; - - #[derive(Clone, Default)] - struct TestCatalogObjectPutPause { - started: Arc, - release: Arc, - } - - impl TestCatalogObjectPutPause { - async fn wait_started(&self) { - self.started.notified().await; - } - - fn release(&self) { - self.release.notify_one(); - } - } - - #[derive(Default)] - struct TestCatalogObjectState { - objects: BTreeMap<(String, String), TestCatalogObjectRecord>, - fail_read_attempts: BTreeMap<(String, String), BTreeSet>, - read_attempts: BTreeMap<(String, String), usize>, - fail_put_attempts: BTreeMap<(String, String), BTreeSet>, - pause_put_attempts: BTreeMap<(String, String), BTreeMap>, - fail_delete_attempts: BTreeMap<(String, String), BTreeSet>, - put_attempts: BTreeMap<(String, String), usize>, - delete_attempts: BTreeMap<(String, String), usize>, - write_lock_acquisitions: BTreeMap<(String, String), usize>, - read_calls: usize, - metadata_calls: usize, - list_calls: usize, - next_etag: u64, - } - - #[derive(Clone)] - struct TestCatalogObjectRecord { - data: Vec, - etag: String, - mod_time: Option, - } - - impl TestCatalogObjectBackend { - async fn seed_object(&self, bucket: &str, object: &str, data: Vec) { - self.seed_object_with_mod_time(bucket, object, data, Some(OffsetDateTime::UNIX_EPOCH)) - .await; - } - - async fn seed_object_with_mod_time(&self, bucket: &str, object: &str, data: Vec, mod_time: Option) { - let mut state = self.state.lock().await; - let etag = state.next_etag(); - state - .objects - .insert((bucket.to_string(), object.to_string()), TestCatalogObjectRecord { data, etag, mod_time }); - } - - async fn fail_put_attempt(&self, bucket: &str, object: &str, attempt: usize) { - let mut state = self.state.lock().await; - state - .fail_put_attempts - .entry((bucket.to_string(), object.to_string())) - .or_default() - .insert(attempt); - } - - async fn fail_delete_attempt(&self, bucket: &str, object: &str, attempt: usize) { - let mut state = self.state.lock().await; - state - .fail_delete_attempts - .entry((bucket.to_string(), object.to_string())) - .or_default() - .insert(attempt); - } - - async fn list_call_count(&self) -> usize { - self.state.lock().await.list_calls - } - - async fn read_call_count(&self) -> usize { - self.state.lock().await.read_calls - } - - async fn metadata_call_count(&self) -> usize { - self.state.lock().await.metadata_calls - } - - async fn reset_call_counts(&self) { - let mut state = self.state.lock().await; - state.read_calls = 0; - state.metadata_calls = 0; - state.list_calls = 0; - } - - async fn write_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize { - self.state - .lock() - .await - .write_lock_acquisitions - .get(&(bucket.to_string(), object.to_string())) - .copied() - .unwrap_or_default() - } - - async fn fail_next_read(&self, bucket: &str, object: &str) { - let mut state = self.state.lock().await; - let key = (bucket.to_string(), object.to_string()); - let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1; - state.fail_read_attempts.entry(key).or_default().insert(next_attempt); - } - - async fn fail_next_put(&self, bucket: &str, object: &str) { - let mut state = self.state.lock().await; - let key = (bucket.to_string(), object.to_string()); - let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1; - state.fail_put_attempts.entry(key).or_default().insert(next_attempt); - } - - async fn pause_next_put(&self, bucket: &str, object: &str) -> TestCatalogObjectPutPause { - let mut state = self.state.lock().await; - let key = (bucket.to_string(), object.to_string()); - let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1; - let pause = TestCatalogObjectPutPause::default(); - state - .pause_put_attempts - .entry(key) - .or_default() - .insert(next_attempt, pause.clone()); - pause - } - } - - impl TestCatalogObjectState { - fn next_etag(&mut self) -> String { - self.next_etag += 1; - format!("etag-{}", self.next_etag) - } - } - - fn maintenance_object_report<'a>( - report: &'a TableMetadataMaintenanceReport, - metadata_location: &str, - ) -> &'a TableMetadataMaintenanceObjectReport { - report - .object_reports - .iter() - .find(|object| object.metadata_location == metadata_location) - .expect("metadata maintenance object report should exist") - } - - fn snapshot_expiration_report( - report: &TableSnapshotExpirationReport, - snapshot_id: i64, - ) -> &TableSnapshotExpirationSnapshotReport { - report - .snapshot_reports - .iter() - .find(|snapshot| snapshot.snapshot_id == Some(snapshot_id)) - .expect("snapshot expiration report should include the snapshot") - } - - fn compaction_snapshot_report(report: &TableCompactionPlanningReport, snapshot_id: i64) -> &TableCompactionSnapshotReport { - report - .snapshot_reports - .iter() - .find(|snapshot| snapshot.snapshot_id == Some(snapshot_id)) - .expect("compaction planning report should include the snapshot") - } - - fn object_cleanup_report<'a>( - report: &'a TableMetadataMaintenanceReport, - object_location: &str, - ) -> &'a TableMetadataMaintenanceObjectCleanupReport { - report - .object_cleanup_reports - .iter() - .find(|object| object.object_location == object_location) - .expect("metadata maintenance object cleanup report should exist") - } - - fn manifest_list_avro_bytes(manifest_paths: &[&str]) -> Vec { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "manifest_file", - "fields": [ - {"name": "manifest_path", "type": "string"}, - {"name": "partition_spec_id", "type": "int"}, - {"name": "sequence_number", "type": "long"}, - {"name": "added_snapshot_id", "type": "long"} - ] - } - "#, - ) - .expect("manifest list avro schema should parse"); - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - for manifest_path in manifest_paths { - writer - .append(apache_avro::types::Value::Record(vec![ - ( - "manifest_path".to_string(), - apache_avro::types::Value::String((*manifest_path).to_string()), - ), - ("partition_spec_id".to_string(), apache_avro::types::Value::Int(0)), - ("sequence_number".to_string(), apache_avro::types::Value::Long(7)), - ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(20)), - ])) - .expect("manifest list record should append"); - } - writer.into_inner().expect("manifest list avro bytes should flush") - } - - fn manifest_avro_bytes(files: &[(&str, i32)]) -> Vec { - manifest_avro_bytes_with_status( - &files - .iter() - .map(|(file_path, content)| (*file_path, *content, 1)) - .collect::>(), - ) - } - - fn manifest_avro_bytes_with_status(files: &[(&str, i32, i32)]) -> Vec { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "manifest_entry", - "fields": [ - {"name": "status", "type": "int"}, - {"name": "snapshot_id", "type": "long"}, - {"name": "sequence_number", "type": "long"}, - {"name": "file_sequence_number", "type": "long"}, - { - "name": "data_file", - "type": { - "type": "record", - "name": "data_file", - "fields": [ - {"name": "content", "type": "int"}, - {"name": "file_path", "type": "string"}, - {"name": "record_count", "type": "long"}, - {"name": "file_size_in_bytes", "type": "long"} - ] - } - } - ] - } - "#, - ) - .expect("manifest avro schema should parse"); - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - for (file_path, content, status) in files { - writer - .append(apache_avro::types::Value::Record(vec![ - ("status".to_string(), apache_avro::types::Value::Int(*status)), - ("snapshot_id".to_string(), apache_avro::types::Value::Long(20)), - ("sequence_number".to_string(), apache_avro::types::Value::Long(7)), - ("file_sequence_number".to_string(), apache_avro::types::Value::Long(7)), - ( - "data_file".to_string(), - apache_avro::types::Value::Record(vec![ - ("content".to_string(), apache_avro::types::Value::Int(*content)), - ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), - ("record_count".to_string(), apache_avro::types::Value::Long(1)), - ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), - ]), - ), - ])) - .expect("manifest record should append"); - } - writer.into_inner().expect("manifest avro bytes should flush") - } - - fn manifest_avro_bytes_with_dt_partition(files: &[(&str, i32, &str)]) -> Vec { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "manifest_entry", - "fields": [ - {"name": "status", "type": "int"}, - {"name": "snapshot_id", "type": "long"}, - {"name": "sequence_number", "type": "long"}, - {"name": "file_sequence_number", "type": "long"}, - { - "name": "data_file", - "type": { - "type": "record", - "name": "data_file", - "fields": [ - {"name": "content", "type": "int"}, - {"name": "file_path", "type": "string"}, - {"name": "partition", "type": {"type": "record", "name": "partition", "fields": [ - {"name": "dt", "type": "string"} - ]}}, - {"name": "record_count", "type": "long"}, - {"name": "file_size_in_bytes", "type": "long"} - ] - } - } - ] - } - "#, - ) - .expect("partitioned manifest avro schema should parse"); - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - for (file_path, content, partition_value) in files { - writer - .append(apache_avro::types::Value::Record(vec![ - ("status".to_string(), apache_avro::types::Value::Int(1)), - ("snapshot_id".to_string(), apache_avro::types::Value::Long(20)), - ("sequence_number".to_string(), apache_avro::types::Value::Long(7)), - ("file_sequence_number".to_string(), apache_avro::types::Value::Long(7)), - ( - "data_file".to_string(), - apache_avro::types::Value::Record(vec![ - ("content".to_string(), apache_avro::types::Value::Int(*content)), - ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), - ( - "partition".to_string(), - apache_avro::types::Value::Record(vec![( - "dt".to_string(), - apache_avro::types::Value::String((*partition_value).to_string()), - )]), - ), - ("record_count".to_string(), apache_avro::types::Value::Long(1)), - ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), - ]), - ), - ])) - .expect("partitioned manifest record should append"); - } - writer.into_inner().expect("partitioned manifest avro bytes should flush") - } - - fn manifest_avro_bytes_with_sort_order(files: &[(&str, i32, i32)]) -> Vec { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "manifest_entry", - "fields": [ - {"name": "status", "type": "int"}, - {"name": "snapshot_id", "type": "long"}, - {"name": "sequence_number", "type": "long"}, - {"name": "file_sequence_number", "type": "long"}, - { - "name": "data_file", - "type": { - "type": "record", - "name": "data_file", - "fields": [ - {"name": "content", "type": "int"}, - {"name": "file_path", "type": "string"}, - {"name": "record_count", "type": "long"}, - {"name": "file_size_in_bytes", "type": "long"}, - {"name": "sort_order_id", "type": ["null", "int"], "default": null} - ] - } - } - ] - } - "#, - ) - .expect("sort-order manifest avro schema should parse"); - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - for (file_path, content, sort_order_id) in files { - writer - .append(apache_avro::types::Value::Record(vec![ - ("status".to_string(), apache_avro::types::Value::Int(1)), - ("snapshot_id".to_string(), apache_avro::types::Value::Long(20)), - ("sequence_number".to_string(), apache_avro::types::Value::Long(7)), - ("file_sequence_number".to_string(), apache_avro::types::Value::Long(7)), - ( - "data_file".to_string(), - apache_avro::types::Value::Record(vec![ - ("content".to_string(), apache_avro::types::Value::Int(*content)), - ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), - ("record_count".to_string(), apache_avro::types::Value::Long(1)), - ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), - ( - "sort_order_id".to_string(), - apache_avro::types::Value::Union(1, Box::new(apache_avro::types::Value::Int(*sort_order_id))), - ), - ]), - ), - ])) - .expect("sort-order manifest record should append"); - } - writer.into_inner().expect("sort-order manifest avro bytes should flush") - } - - fn parquet_i32_bytes(values: &[i32]) -> Vec { - let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); - let batch = RecordBatch::try_new(Arc::clone(&schema) as SchemaRef, vec![Arc::new(Int32Array::from(values.to_vec()))]) - .expect("parquet test batch should build"); - let mut bytes = Vec::new(); - { - let mut writer = ArrowWriter::try_new(&mut bytes, schema, None).expect("parquet writer should build"); - writer.write(&batch).expect("parquet batch should write"); - writer.close().expect("parquet writer should close"); - } - bytes - } - - fn parquet_i64_bytes(values: &[i64]) -> Vec { - let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); - let batch = RecordBatch::try_new(Arc::clone(&schema) as SchemaRef, vec![Arc::new(Int64Array::from(values.to_vec()))]) - .expect("parquet test batch should build"); - let mut bytes = Vec::new(); - { - let mut writer = ArrowWriter::try_new(&mut bytes, schema, None).expect("parquet writer should build"); - writer.write(&batch).expect("parquet batch should write"); - writer.close().expect("parquet writer should close"); - } - bytes - } - - fn parquet_i32_values(data: Vec) -> Vec { - let reader = ParquetRecordBatchReaderBuilder::try_new(bytes::Bytes::from(data)) - .expect("parquet reader should build") - .build() - .expect("parquet batches should build"); - let mut values = Vec::new(); - for batch in reader { - let batch = batch.expect("parquet batch should read"); - let column = batch - .column(0) - .as_any() - .downcast_ref::() - .expect("first parquet column should be int32"); - values.extend((0..column.len()).map(|index| column.value(index))); - } - values - } - - #[async_trait::async_trait] - impl TableCatalogObjectBackend for TestCatalogObjectBackend { - async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - let mut state = self.state.lock().await; - state.read_calls += 1; - let key = (bucket.to_string(), object.to_string()); - let attempt = { - let attempts = state.read_attempts.entry(key.clone()).or_default(); - *attempts += 1; - *attempts - }; - if state - .fail_read_attempts - .get(&key) - .is_some_and(|attempts| attempts.contains(&attempt)) - { - return Err(TableCatalogStoreError::Internal(format!( - "injected read failure for {object} attempt {attempt}" - ))); - } - Ok(state.objects.get(&key).map(|record| TableCatalogObject { - data: record.data.clone(), - etag: Some(record.etag.clone()), - mod_time: record.mod_time, - })) - } - - async fn object_metadata( - &self, - bucket: &str, - object: &str, - ) -> TableCatalogStoreResult> { - let mut state = self.state.lock().await; - state.metadata_calls += 1; - Ok(state - .objects - .get(&(bucket.to_string(), object.to_string())) - .map(|record| TableCatalogObjectMetadata { - etag: Some(record.etag.clone()), - mod_time: record.mod_time, - })) - } - - async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { - let state = self.state.lock().await; - Ok(state.objects.contains_key(&(bucket.to_string(), object.to_string()))) - } - - async fn put_object( - &self, - bucket: &str, - object: &str, - data: Vec, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> { - let key = (bucket.to_string(), object.to_string()); - let pause = { - let mut state = self.state.lock().await; - let attempt = { - let attempts = state.put_attempts.entry(key.clone()).or_default(); - *attempts += 1; - *attempts - }; - if state - .fail_put_attempts - .get(&key) - .is_some_and(|attempts| attempts.contains(&attempt)) - { - return Err(TableCatalogStoreError::Internal(format!( - "injected put failure for {object} attempt {attempt}" - ))); - } - state - .pause_put_attempts - .get_mut(&key) - .and_then(|attempts| attempts.remove(&attempt)) - }; - if let Some(pause) = pause { - pause.started.notify_one(); - pause.release.notified().await; - } - - let mut state = self.state.lock().await; - match precondition { - TableCatalogPutPrecondition::IfAbsent if state.objects.contains_key(&key) => { - return Err(TableCatalogStoreError::Conflict(format!("object already exists: {object}"))); - } - TableCatalogPutPrecondition::IfMatch(expected) => { - let Some(current) = state.objects.get(&key) else { - return Err(TableCatalogStoreError::Conflict(format!("object is missing: {object}"))); - }; - if current.etag != expected { - return Err(TableCatalogStoreError::Conflict(format!("object changed: {object}"))); - } - } - _ => {} - } - - let etag = state.next_etag(); - state.objects.insert( - key, - TestCatalogObjectRecord { - data, - etag, - mod_time: Some(OffsetDateTime::now_utc()), - }, - ); - Ok(()) - } - - async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { - let mut state = self.state.lock().await; - let key = (bucket.to_string(), object.to_string()); - let attempt = { - let attempts = state.delete_attempts.entry(key.clone()).or_default(); - *attempts += 1; - *attempts - }; - if state - .fail_delete_attempts - .get(&key) - .is_some_and(|attempts| attempts.contains(&attempt)) - { - return Err(TableCatalogStoreError::Internal(format!( - "injected delete failure for {object} attempt {attempt}" - ))); - } - state.objects.remove(&key); - Ok(()) - } - - async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult> { - let mut state = self.state.lock().await; - state.list_calls += 1; - Ok(state - .objects - .keys() - .filter(|(entry_bucket, object)| entry_bucket == bucket && object.starts_with(prefix)) - .map(|(_, object)| object.clone()) - .collect()) - } - - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - { - let mut state = self.state.lock().await; - *state - .write_lock_acquisitions - .entry((bucket.to_string(), object.to_string())) - .or_default() += 1; - } - let lock = { - let mut locks = self.locks.lock().await; - locks - .entry((bucket.to_string(), object.to_string())) - .or_insert_with(|| std::sync::Arc::new(tokio::sync::Mutex::new(()))) - .clone() - }; - Ok(Box::new(lock.lock_owned().await)) - } - } - - fn test_bucket_entry(bucket: &str) -> TableBucketEntry { - TableBucketEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - catalog_type: TABLE_BUCKET_CATALOG_TYPE.to_string(), - warehouse_root: format!("s3://{bucket}/"), - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - } - } - - fn test_namespace_entry(bucket: &str, namespace: &Namespace) -> NamespaceEntry { - NamespaceEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - namespace_id: namespace.storage_id(), - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - } - } - - fn test_table_entry(bucket: &str, namespace: &Namespace, table: &IdentifierSegment, metadata_location: String) -> TableEntry { - TableEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: format!("s3://{bucket}/tables/table-id"), - metadata_location, - version_token: "token-v1".to_string(), - generation: 1, - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - } - } - - fn test_view_entry(bucket: &str, namespace: &Namespace, view: &IdentifierSegment, metadata_location: String) -> ViewEntry { - ViewEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.as_str().to_string(), - view_id: "view-id".to_string(), - view_uuid: "view-uuid".to_string(), - format: "ICEBERG_VIEW".to_string(), - format_version: 1, - warehouse_location: format!("s3://{bucket}/views/view-id"), - metadata_location, - version_token: "token-v1".to_string(), - generation: 1, - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - } - } - - async fn seed_catalog_list_entries(store: &S, bucket: &str, namespace: &Namespace) - where - S: TableCatalogStore + ?Sized, - { - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - for namespace in [ - Namespace::parse("analytics").expect("namespace should parse"), - namespace.clone(), - ] { - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("namespace should be created"); - } - for name in ["alpha", "beta"] { - let identifier = IdentifierSegment::parse(name).expect("table and view name should parse"); - let metadata_location = default_table_metadata_file_path(namespace, &identifier, "00001.metadata.json"); - let mut table = test_table_entry(bucket, namespace, &identifier, metadata_location.clone()); - table.table_id = format!("table-{name}"); - table.table_uuid = format!("table-uuid-{name}"); - table.warehouse_location = format!("s3://{bucket}/tables/table-{name}"); - store.create_table(table).await.expect("table should be created"); - - let mut view = test_view_entry(bucket, namespace, &identifier, metadata_location); - view.view_id = format!("view-{name}"); - view.view_uuid = format!("view-uuid-{name}"); - view.warehouse_location = format!("s3://{bucket}/views/view-{name}"); - store.create_view(view).await.expect("view should be created"); - } - } - - async fn seed_table_for_metadata_maintenance( - store: &ObjectTableCatalogStore, - bucket: &str, - namespace: &Namespace, - table: &IdentifierSegment, - current_metadata: String, - ) { - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store.create_namespace(test_namespace_entry(bucket, namespace)).await.unwrap(); - store - .create_table(test_table_entry(bucket, namespace, table, current_metadata)) - .await - .unwrap(); - store.backfill_table_warehouse_index(bucket).await.unwrap(); - store.backend.reset_call_counts().await; - } - - async fn seed_quarantined_table_maintenance( - store: &ObjectTableCatalogStore, - backend: &TestCatalogObjectBackend, - bucket: &str, - now: OffsetDateTime, - next_retry_after: Option, - ) -> (Namespace, IdentifierSegment, TableMetadataMaintenanceReport) { - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - max_retry_attempts: 2, - retry_initial_backoff_seconds: 60, - retry_max_backoff_seconds: 300, - quarantine_enabled: true, - quarantine_retention_seconds: 86_400, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - let mut failed = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - failed.job.status = TableMetadataMaintenanceJobStatus::Failed; - failed.job.failure_reason = Some("quarantine retained failed cleanup candidates".to_string()); - failed.job.max_retry_attempts = 2; - failed.job.next_retry_after = next_retry_after.map(maintenance_timestamp); - failed.job.quarantine_enabled = true; - failed.job.quarantine_retention_seconds = 86_400; - failed.job.quarantined_object_count = 2; - failed.job.finished_at = Some(maintenance_timestamp(now - Duration::seconds(10))); - store - .put_table_metadata_maintenance_report(&failed) - .await - .expect("failed maintenance report should be seeded"); - (namespace, table, failed) - } - - #[tokio::test] - async fn object_table_catalog_store_writes_catalog_entries_to_internal_meta_bucket() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - assert!(!store.warehouse_index_ready(bucket).await.unwrap()); - - let state = backend.state.lock().await; - let object_buckets = state - .objects - .keys() - .map(|(bucket, _)| bucket.as_str()) - .collect::>(); - - assert_eq!(object_buckets, BTreeSet::from([RUSTFS_META_BUCKET])); - } - - #[tokio::test] - async fn configured_table_catalog_store_uses_durable_strong_snapshot() { - let backend = TestCatalogObjectBackend::default(); - let store = ConfiguredTableCatalogStore::new(backend.clone(), TableCatalogBackingMode::DurableStrong); - let bucket = "analytics"; - - assert_eq!(store.backing_mode(), TableCatalogBackingMode::DurableStrong); - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - - let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); - assert!(backend.object_exists(RUSTFS_META_BUCKET, &snapshot_path).await.unwrap()); - - let reloaded = ConfiguredTableCatalogStore::new(backend.clone(), TableCatalogBackingMode::DurableStrong); - assert!(reloaded.get_table_bucket(bucket).await.unwrap().is_some()); - } - - #[tokio::test] - async fn object_table_catalog_store_persists_view_entries_and_blocks_non_empty_namespace_drop() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let view = IdentifierSegment::parse("recent_orders").unwrap(); - let current_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); - let next_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_view(test_view_entry(bucket, &namespace, &view, current_metadata.clone())) - .await - .unwrap(); - - assert_eq!(store.list_views(bucket, &namespace.public_name()).await.unwrap()[0].view, "recent_orders"); - assert!( - store - .load_view(bucket, &namespace.public_name(), view.as_str()) - .await - .unwrap() - .is_some() - ); - assert!(matches!( - store.drop_namespace(bucket, &namespace.public_name()).await, - Err(TableCatalogStoreError::Conflict(_)) - )); - - backend - .seed_object( - bucket, - &next_metadata, - serde_json::to_vec(&serde_json::json!({ - "format-version": 1, - "view-uuid": "view-uuid", - "location": format!("s3://{bucket}/views/view-id") - })) - .unwrap(), - ) - .await; - let result = store - .replace_view(ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.as_str().to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: next_metadata.clone(), - }) - .await - .unwrap(); - - assert_eq!(result.view.metadata_location, next_metadata); - assert_eq!(result.view.generation, 2); - assert_ne!(result.view.version_token, "token-v1"); - - store - .drop_view(bucket, &namespace.public_name(), view.as_str()) - .await - .unwrap(); - assert!(store.list_views(bucket, &namespace.public_name()).await.unwrap().is_empty()); - store.drop_namespace(bucket, &namespace.public_name()).await.unwrap(); - } - - #[tokio::test] - async fn maintenance_dry_run_keeps_current_metadata() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let v1 = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let v2 = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &v1, b"{}".to_vec()).await; - backend.seed_object(bucket, &v2, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .unwrap(); - - assert_eq!(report.current_metadata_location, current); - assert!(report.retained_metadata_locations.contains(&report.current_metadata_location)); - assert!(!report.cleanup_candidate_locations.contains(&report.current_metadata_location)); - assert_eq!(report.cleanup_candidate_locations, vec![v1, v2]); - } - - #[tokio::test] - async fn table_data_plane_resource_resolves_registered_warehouse_prefix() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - assert_eq!(backend.list_call_count().await, 0); - - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") - .await - .expect("data-plane resource lookup should succeed") - .expect("object should resolve to the registered table"); - - assert_eq!(resource.table_bucket, bucket); - assert_eq!(resource.namespace, "sales"); - assert_eq!(resource.table, "orders"); - assert_eq!(resource.table_id, "table-id"); - assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); - assert_eq!(resource.catalog_resource_object(), "namespaces/sales/tables/orders"); - assert_eq!(backend.list_call_count().await, 0); - } - - #[tokio::test] - async fn table_data_plane_resource_does_not_match_sibling_prefix() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id-other/data/part-00001.parquet") - .await - .expect("data-plane resource lookup should succeed"); - - assert!(resource.is_none()); - assert_eq!(backend.list_call_count().await, 0); - } - - #[tokio::test] - async fn table_data_plane_resource_prefers_longest_registered_warehouse_prefix() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let parent_table = IdentifierSegment::parse("orders").unwrap(); - let child_table = IdentifierSegment::parse("orders_child").unwrap(); - let current = default_table_metadata_file_path(&namespace, &parent_table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent_table, current.clone()).await; - let mut child_entry = test_table_entry(bucket, &namespace, &child_table, current); - child_entry.table_id = "table-id-child".to_string(); - child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); - store.create_table(child_entry).await.unwrap(); - - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/child/data/part-00001.parquet") - .await - .expect("data-plane resource lookup should succeed") - .expect("object should resolve to the child table"); - - assert_eq!(resource.table, "orders_child"); - assert_eq!(resource.table_id, "table-id-child"); - assert_eq!(resource.warehouse_object_prefix, "tables/table-id/child/"); - assert_eq!(resource.catalog_resource_object(), "namespaces/sales/tables/orders_child"); - assert_eq!(backend.list_call_count().await, 0); - } - - #[tokio::test] - async fn table_data_plane_resource_skips_stale_deeper_index_and_matches_parent() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - let stale_prefix = "tables/table-id/child/"; - let stale_index = TableWarehouseIndexEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - table_id: "table-id".to_string(), - warehouse_object_prefix: stale_prefix.to_string(), - state: TableCatalogEntryState::Active, - }; - store - .write_entry( - store.catalog_bucket(), - &store.paths.warehouse_index_entry_path(bucket, stale_prefix), - &stale_index, - TableCatalogPutPrecondition::Any, - ) - .await - .unwrap(); - - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/child/data/part-00001.parquet") - .await - .expect("stale deeper index should not fail lookup") - .expect("parent table should still protect the object"); - - assert_eq!(resource.table, "orders"); - assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); - assert_eq!(backend.list_call_count().await, 0); - } - - #[tokio::test] - async fn object_table_catalog_store_rejects_duplicate_warehouse_prefix() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let first_table = IdentifierSegment::parse("orders").unwrap(); - let second_table = IdentifierSegment::parse("returns").unwrap(); - let current = default_table_metadata_file_path(&namespace, &first_table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &first_table, current.clone()).await; - let mut second_entry = test_table_entry(bucket, &namespace, &second_table, current); - second_entry.table_id = "second-table-id".to_string(); - second_entry.warehouse_location = format!("s3://{bucket}/tables/table-id"); - - let error = store.create_table(second_entry).await.unwrap_err(); - - assert!(matches!( - error, - TableCatalogStoreError::Conflict(message) if message.contains("warehouse location is already registered") - )); - } - - #[tokio::test] - async fn table_data_plane_resource_fails_closed_for_missing_indexed_table() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let prefix = "tables/missing-table/"; - let index = TableWarehouseIndexEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: "orders".to_string(), - table_id: "missing-table-id".to_string(), - warehouse_object_prefix: prefix.to_string(), - state: TableCatalogEntryState::Active, - }; - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store.backfill_table_warehouse_index(bucket).await.unwrap(); - store - .write_entry( - store.catalog_bucket(), - &store.paths.warehouse_index_entry_path(bucket, prefix), - &index, - TableCatalogPutPrecondition::Any, - ) - .await - .unwrap(); - - let error = table_data_plane_resource_for_object(&store, bucket, "tables/missing-table/data/part-00001.parquet") - .await - .unwrap_err(); - - assert!(matches!( - error, - TableCatalogStoreError::Internal(message) if message.contains("referenced table entry is missing") - )); - } - - #[tokio::test] - async fn object_table_catalog_store_replaces_stale_warehouse_index_on_create() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let prefix = "tables/shared-table/"; - let index_path = store.paths.warehouse_index_entry_path(bucket, prefix); - let stale_index = TableWarehouseIndexEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: "missing_orders".to_string(), - table_id: "missing-table-id".to_string(), - warehouse_object_prefix: prefix.to_string(), - state: TableCatalogEntryState::Active, - }; - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .write_entry(store.catalog_bucket(), &index_path, &stale_index, TableCatalogPutPrecondition::Any) - .await - .unwrap(); - - let mut entry = test_table_entry(bucket, &namespace, &table, current); - entry.warehouse_location = format!("s3://{bucket}/tables/shared-table"); - store - .create_table(entry) - .await - .expect("stale warehouse index should be repaired before reserving the prefix"); - - let (index, _) = store - .read_entry::(store.catalog_bucket(), &index_path) - .await - .unwrap() - .expect("repaired index should exist"); - assert_eq!(index.table, "orders"); - assert_eq!(index.table_id, "table-id"); - } - - #[tokio::test] - async fn table_data_plane_resource_falls_back_to_scan_without_index_state() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let bucket_entry = test_bucket_entry(bucket); - let namespace_entry = test_namespace_entry(bucket, &namespace); - let table_entry = test_table_entry(bucket, &namespace, &table, current); - - store - .write_entry( - store.catalog_bucket(), - &store.paths.table_bucket_entry_path(bucket), - &bucket_entry, - TableCatalogPutPrecondition::Any, - ) - .await - .expect("table bucket entry should be seeded"); - store - .write_entry( - store.catalog_bucket(), - &store.paths.namespace_entry_path(bucket, &namespace), - &namespace_entry, - TableCatalogPutPrecondition::Any, - ) - .await - .unwrap(); - store - .write_entry( - store.catalog_bucket(), - &store.paths.table_entry_path(bucket, &namespace, &table), - &table_entry, - TableCatalogPutPrecondition::Any, - ) - .await - .unwrap(); - - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") - .await - .expect("legacy catalog lookup should fall back to scanning") - .expect("legacy table entry should resolve"); - - assert_eq!(resource.table, "orders"); - assert!(backend.list_call_count().await > 0); - assert!(store.warehouse_index_ready(bucket).await.unwrap()); - - backend.reset_call_counts().await; - let indexed_resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00002.parquet") - .await - .expect("backfilled index lookup should succeed") - .expect("backfilled table entry should resolve"); - - assert_eq!(indexed_resource.table, "orders"); - assert_eq!(backend.list_call_count().await, 0); - assert!(backend.read_call_count().await <= 5); - } - - #[tokio::test] - async fn object_table_catalog_store_backfill_skips_table_deleted_after_listing() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let bucket_entry = test_bucket_entry(bucket); - let namespace_entry = test_namespace_entry(bucket, &namespace); - let table_entry = test_table_entry(bucket, &namespace, &table, current); - let table_path = store.paths.table_entry_path(bucket, &namespace, &table); - let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); - - store - .write_entry( - store.catalog_bucket(), - &store.paths.table_bucket_entry_path(bucket), - &bucket_entry, - TableCatalogPutPrecondition::Any, - ) - .await - .unwrap(); - store - .write_entry( - store.catalog_bucket(), - &store.paths.namespace_entry_path(bucket, &namespace), - &namespace_entry, - TableCatalogPutPrecondition::Any, - ) - .await - .expect("namespace entry should be seeded"); - store - .write_entry(store.catalog_bucket(), &table_path, &table_entry, TableCatalogPutPrecondition::Any) - .await - .expect("table entry should be seeded without an index"); - - let listed = store - .list_tables(bucket, &namespace.public_name()) - .await - .expect("table listing should succeed"); - assert_eq!(listed.len(), 1); - backend - .delete_object(RUSTFS_META_BUCKET, &table_path) - .await - .expect("listed table entry should be deleted before backfill"); - - for table in listed { - store - .backfill_active_table_warehouse_index(&table.table_bucket, &table.namespace, &table.table) - .await - .expect("backfill should skip a table deleted after listing"); - } - - assert!( - store - .read_entry::(store.catalog_bucket(), &index_path) - .await - .expect("warehouse index lookup should succeed") - .is_none() - ); - } - - #[tokio::test] - async fn object_catalog_pagination_bounds_reads_and_covers_rest_resources() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let namespace_name = namespace.public_name(); - let one = NonZeroUsize::new(1).expect("page size should be non-zero"); - - seed_catalog_list_entries(&store, bucket, &namespace).await; - - let namespace_page = store - .list_namespaces_page(bucket, None, one) - .await - .expect("first namespace page should load"); - assert_eq!(namespace_page.entries[0].namespace, "analytics"); - assert!( - namespace_page - .next_cursor - .as_deref() - .is_some_and(|cursor| cursor.starts_with(OBJECT_CATALOG_LIST_CURSOR_PREFIX)) - ); - let namespace_page = store - .list_namespaces_page(bucket, namespace_page.next_cursor.as_deref(), one) - .await - .expect("second namespace page should load"); - assert_eq!(namespace_page.entries[0].namespace, "sales"); - assert!(namespace_page.next_cursor.is_none()); - - backend.reset_call_counts().await; - let table_page = store - .list_tables_page(bucket, &namespace_name, None, one) - .await - .expect("first table page should load"); - assert_eq!(table_page.entries[0].table, "alpha"); - assert_eq!(backend.read_call_count().await, 1); - let table_page = store - .list_tables_page(bucket, &namespace_name, table_page.next_cursor.as_deref(), one) - .await - .expect("second table page should load"); - assert_eq!(table_page.entries[0].table, "beta"); - assert!(table_page.next_cursor.is_none()); - - let view_page = store - .list_views_page(bucket, &namespace_name, None, one) - .await - .expect("first view page should load"); - assert_eq!(view_page.entries[0].view, "alpha"); - let view_page = store - .list_views_page(bucket, &namespace_name, view_page.next_cursor.as_deref(), one) - .await - .expect("second view page should load"); - assert_eq!(view_page.entries[0].view, "beta"); - assert!(view_page.next_cursor.is_none()); - - let exact_page = store - .list_tables_page(bucket, &namespace_name, None, NonZeroUsize::new(2).expect("page size should be non-zero")) - .await - .expect("exact table page should load"); - assert_eq!(exact_page.entries.len(), 2); - assert!(exact_page.next_cursor.is_none()); - assert!(matches!( - store - .list_tables_page(bucket, &namespace_name, Some("strong:alpha"), one) - .await, - Err(TableCatalogStoreError::Invalid(_)) - )); - } - - #[tokio::test] - async fn object_catalog_pagination_bounds_sparse_namespace_scans() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let prefix = store.paths.namespace_entries_prefix(bucket); - - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("namespace should be created"); - for index in 0..TABLE_CATALOG_LIST_MAX_KEYS { - backend - .seed_object(RUSTFS_META_BUCKET, &format!("{prefix}0000-spacer/{index:04}.json"), Vec::new()) - .await; - } - backend.reset_call_counts().await; - - let one = NonZeroUsize::new(1).expect("page size should be non-zero"); - let first = store - .list_namespaces_page(bucket, None, one) - .await - .expect("sparse first page should load"); - assert!(first.entries.is_empty()); - assert_eq!(backend.list_call_count().await, 1); - let cursor = first.next_cursor.expect("truncated sparse page should have a cursor"); - assert!(cursor.starts_with(OBJECT_CATALOG_LIST_CURSOR_PREFIX)); - assert!(!cursor.ends_with(NAMESPACE_ENTRY_FILE)); - - let second = store - .list_namespaces_page(bucket, Some(&cursor), one) - .await - .expect("sparse continuation page should load"); - assert_eq!(second.entries.len(), 1); - assert_eq!(second.entries[0].namespace, namespace.public_name()); - assert!(second.next_cursor.is_none()); - assert_eq!(backend.list_call_count().await, 2); - } - - #[tokio::test] - async fn object_table_catalog_store_rolls_back_warehouse_index_when_table_entry_write_fails() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let failed_table = IdentifierSegment::parse("failed_orders").unwrap(); - let next_table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &failed_table, "00001.metadata.json"); - let failed_table_path = store.paths.table_entry_path(bucket, &namespace, &failed_table); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - backend.fail_put_attempt(RUSTFS_META_BUCKET, &failed_table_path, 1).await; - - let mut failed_entry = test_table_entry(bucket, &namespace, &failed_table, current.clone()); - failed_entry.table_id = "failed-table-id".to_string(); - failed_entry.warehouse_location = format!("s3://{bucket}/tables/shared-table"); - let error = store.create_table(failed_entry).await.unwrap_err(); - assert!(matches!(error, TableCatalogStoreError::Internal(_))); - - let mut next_entry = test_table_entry(bucket, &namespace, &next_table, current); - next_entry.table_id = "next-table-id".to_string(); - next_entry.warehouse_location = format!("s3://{bucket}/tables/shared-table"); - store - .create_table(next_entry) - .await - .expect("rolled back warehouse index should not block the next table"); - } - - #[tokio::test] - async fn object_table_catalog_store_restores_table_when_drop_index_delete_fails() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - backend.fail_delete_attempt(RUSTFS_META_BUCKET, &index_path, 1).await; - - let error = store - .drop_table(bucket, &namespace.public_name(), table.as_str()) - .await - .unwrap_err(); - - assert!(matches!(error, TableCatalogStoreError::Internal(_))); - let restored = store - .load_table(bucket, &namespace.public_name(), table.as_str()) - .await - .expect("restored table lookup should succeed") - .expect("table entry should be restored"); - assert_eq!(restored.table_id, "table-id"); - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") - .await - .expect("restored table data-plane lookup should succeed") - .expect("restored table should keep data-plane protection"); - assert_eq!(resource.table, "orders"); - } - - #[tokio::test] - async fn table_data_plane_resource_bounds_deep_object_index_reads() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - backend.reset_call_counts().await; - - let deep_suffix = (0..100).map(|index| format!("level-{index}/")).collect::(); - let object = format!("tables/table-id/{deep_suffix}part-00001.parquet"); - let resource = table_data_plane_resource_for_object(&store, bucket, &object) - .await - .expect("deep object lookup should succeed") - .expect("deep object should resolve to the table"); - - assert_eq!(resource.table, "orders"); - assert_eq!(backend.list_call_count().await, 0); - assert!(backend.read_call_count().await <= WAREHOUSE_INDEX_MAX_PREFIX_DEPTH + 3); - } - - #[tokio::test] - async fn object_table_catalog_store_rejects_invalid_table_warehouse_location() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - - let mut entry = test_table_entry(bucket, &namespace, &table, current); - entry.warehouse_location = format!("s3://{bucket}/tables/../table-id"); - - let error = store.create_table(entry).await.unwrap_err(); - assert!(matches!( - error, - TableCatalogStoreError::Invalid(message) if message.contains("invalid path segment") - )); - } - - #[tokio::test] - async fn object_table_catalog_store_rejects_deep_table_warehouse_location() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let deep_prefix = (0..=WAREHOUSE_INDEX_MAX_PREFIX_DEPTH) - .map(|index| format!("level-{index}")) - .collect::>() - .join("/"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - - let mut entry = test_table_entry(bucket, &namespace, &table, current); - entry.warehouse_location = format!("s3://{bucket}/{deep_prefix}"); - - let error = store.create_table(entry).await.unwrap_err(); - assert!(matches!( - error, - TableCatalogStoreError::Invalid(message) if message.contains("maximum prefix depth") - )); - } - - #[tokio::test] - async fn object_table_catalog_store_rejects_invalid_view_warehouse_location() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); - let current = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); - - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("namespace should be created"); - - let mut entry = test_view_entry(bucket, &namespace, &view, current); - entry.warehouse_location = format!("s3://{bucket}/views/../view-id"); - - let error = store.create_view(entry).await.unwrap_err(); - assert!(matches!( - error, - TableCatalogStoreError::Invalid(message) if message.contains("invalid path segment") - )); - } - - #[tokio::test] - async fn object_table_catalog_store_allows_deep_view_warehouse_location() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let view = IdentifierSegment::parse("recent_orders").unwrap(); - let current = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); - let next = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); - let deep_prefix = (0..=WAREHOUSE_INDEX_MAX_PREFIX_DEPTH) - .map(|index| format!("level-{index}")) - .collect::>() - .join("/"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - - let mut entry = test_view_entry(bucket, &namespace, &view, current.clone()); - entry.warehouse_location = format!("s3://{bucket}/{deep_prefix}"); - store - .create_view(entry) - .await - .expect("view warehouse location should not inherit table index depth limits"); - - let relocated_prefix = format!("{deep_prefix}/relocated"); - backend - .seed_object( - bucket, - &next, - serde_json::to_vec(&serde_json::json!({ - "format-version": 1, - "view-uuid": "view-uuid", - "location": format!("s3://{bucket}/{relocated_prefix}") - })) - .expect("view metadata should serialize"), - ) - .await; - - let result = store - .replace_view(ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.as_str().to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current, - new_metadata_location: next, - }) - .await - .expect("view metadata location should not inherit table index depth limits"); - - assert_eq!(result.view.warehouse_location, format!("s3://{bucket}/{relocated_prefix}")); - } - - #[tokio::test] - async fn table_data_plane_resource_skips_invalid_warehouse_locations() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let invalid_table = IdentifierSegment::parse("bad_orders").unwrap(); - let valid_table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &valid_table, "00001.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - - let mut invalid_entry = test_table_entry(bucket, &namespace, &invalid_table, current.clone()); - invalid_entry.table_id = "bad-table-id".to_string(); - invalid_entry.warehouse_location = format!("s3://{bucket}/"); - let invalid_path = store.paths.table_entry_path(bucket, &namespace, &invalid_table); - store - .write_entry( - store.catalog_bucket(), - &invalid_path, - &invalid_entry, - TableCatalogPutPrecondition::IfAbsent, - ) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &valid_table, current)) - .await - .unwrap(); - - let unrelated = table_data_plane_resource_for_object(&store, bucket, "ordinary/object.parquet") - .await - .expect("invalid table warehouse location should not deny unrelated object lookup"); - assert!(unrelated.is_none()); - - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") - .await - .expect("invalid table warehouse location should not block a later valid match") - .expect("valid table warehouse object should resolve to the table"); - assert_eq!(resource.table, "orders"); - assert_eq!(resource.table_id, "table-id"); - assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); - } - - #[tokio::test] - async fn maintenance_dry_run_reports_job_context_and_deletable_candidates() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let fresh = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - backend - .seed_object_with_mod_time(bucket, &fresh, b"{}".to_vec(), Some(OffsetDateTime::now_utc())) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .unwrap(); - - assert_eq!(report.job.table_bucket, bucket); - assert_eq!(report.job.namespace, "sales"); - assert_eq!(report.job.table, "orders"); - assert_eq!(report.job.table_id, "table-id"); - assert_eq!(report.job.operation, TableMetadataMaintenanceOperation::DryRun); - assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Successful); - assert_eq!(report.job.deleted_metadata_file_count, 0); - assert_eq!(report.job.current_generation, 1); - assert_eq!(report.job.safety_window_seconds, TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS); - assert!(!report.job.job_id.is_empty()); - assert!(report.job.cleanup_watermark_unix_seconds <= OffsetDateTime::now_utc().unix_timestamp()); - assert_eq!(report.cleanup_candidate_locations, vec![old.clone(), fresh]); - assert_eq!(report.deletable_metadata_locations, vec![old]); - } - - #[tokio::test] - async fn maintenance_dry_run_explains_metadata_reachability() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let logged = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let fresh = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let old = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - let recent = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00005.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &logged, b"{}".to_vec()).await; - backend.seed_object(bucket, &recent, b"{}".to_vec()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object_with_mod_time(bucket, &fresh, b"{}".to_vec(), Some(OffsetDateTime::now_utc())) - .await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [ - { - "timestamp-ms": 1, - "metadata-file": logged - } - ] - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 2) - .await - .unwrap(); - - assert_eq!(report.job.planned_metadata_file_count, 5); - assert_eq!(report.job.retained_metadata_file_count, 3); - assert_eq!(report.job.cleanup_candidate_count, 2); - assert_eq!(report.job.deletable_metadata_file_count, 1); - assert_eq!( - report.job.recommended_actions, - vec![TableMaintenanceRecommendedAction::ReviewAndRunDelete] - ); - - let current_report = maintenance_object_report(&report, ¤t); - assert_eq!(current_report.state, TableMetadataMaintenanceObjectState::Retained); - assert_eq!(current_report.reasons, vec![TableMetadataMaintenanceReason::CurrentMetadata]); - - let logged_report = maintenance_object_report(&report, &logged); - assert_eq!(logged_report.state, TableMetadataMaintenanceObjectState::Retained); - assert_eq!(logged_report.reasons, vec![TableMetadataMaintenanceReason::MetadataLog]); - - let recent_report = maintenance_object_report(&report, &recent); - assert_eq!(recent_report.state, TableMetadataMaintenanceObjectState::Retained); - assert_eq!(recent_report.reasons, vec![TableMetadataMaintenanceReason::RecentMetadata]); - - let old_report = maintenance_object_report(&report, &old); - assert_eq!(old_report.state, TableMetadataMaintenanceObjectState::Deletable); - assert_eq!( - old_report.reasons, - vec![ - TableMetadataMaintenanceReason::NoCurrentReachability, - TableMetadataMaintenanceReason::SafetyWindowSatisfied, - ] - ); - - let fresh_report = maintenance_object_report(&report, &fresh); - assert_eq!(fresh_report.state, TableMetadataMaintenanceObjectState::PendingSafetyWindow); - assert_eq!( - fresh_report.reasons, - vec![ - TableMetadataMaintenanceReason::NoCurrentReachability, - TableMetadataMaintenanceReason::SafetyWindowPending, - ] - ); - } - - #[tokio::test] - async fn maintenance_report_read_back_derives_actions_for_legacy_records() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - let mut report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - report.job.status = TableMetadataMaintenanceJobStatus::Running; - report.job.worker_id = Some("worker-a".to_string()); - report.job.lease_id = "lease-a".to_string(); - report.job.heartbeat_at = Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10))); - - let job_path = store - .paths - .table_maintenance_job_path(bucket, &namespace, &table, "table-id", &report.job.job_id); - let mut legacy_report = serde_json::to_value(&report).expect("legacy report should serialize"); - legacy_report - .get_mut("job") - .and_then(serde_json::Value::as_object_mut) - .expect("legacy report job should be an object") - .remove("recommended-actions"); - store - .write_entry(store.catalog_bucket(), &job_path, &legacy_report, TableCatalogPutPrecondition::Any) - .await - .expect("legacy maintenance report should be seeded"); - - let loaded = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id) - .await - .expect("legacy maintenance report lookup should succeed") - .expect("legacy maintenance report should be returned"); - - assert_eq!( - loaded.job.recommended_actions, - vec![TableMaintenanceRecommendedAction::WaitForActiveWorker] - ); - } - - #[tokio::test] - async fn maintenance_state_is_scoped_to_current_table_identity() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - - let mut first_table = test_table_entry(bucket, &namespace, &table, current.clone()); - first_table.table_id = "table-id-1".to_string(); - store.create_table(first_table).await.unwrap(); - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 7, - delete_enabled: true, - background_enabled: false, - ..Default::default() - }, - ) - .await - .unwrap(); - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .unwrap(); - store.put_table_metadata_maintenance_report(&report).await.unwrap(); - assert!( - store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id) - .await - .unwrap() - .is_some() - ); - - store.drop_table(bucket, "sales", "orders").await.unwrap(); - - let mut second_table = test_table_entry(bucket, &namespace, &table, current); - second_table.table_id = "table-id-2".to_string(); - store.create_table(second_table).await.unwrap(); - - assert_eq!( - store.get_table_maintenance_config(bucket, "sales", "orders").await.unwrap(), - TableMaintenanceConfig::default() - ); - assert!( - store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id) - .await - .unwrap() - .is_none() - ); - } - - #[tokio::test] - async fn maintenance_config_rejects_unsupported_config_version() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - - let err = store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION.saturating_add(1), - retain_recent_metadata_files: 1, - delete_enabled: false, - background_enabled: false, - ..Default::default() - }, - ) - .await - .unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Invalid(_)); - } - - #[tokio::test] - async fn maintenance_config_inherits_bucket_default_and_tracks_override_source() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - store - .put_table_bucket_maintenance_config( - bucket, - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 3, - delete_enabled: true, - background_enabled: false, - ..Default::default() - }, - ) - .await - .expect("bucket default maintenance config should persist"); - - let inherited = store - .get_effective_table_maintenance_config(bucket, "sales", "orders") - .await - .expect("effective maintenance config should load"); - - assert_eq!(inherited.source, TableMaintenanceConfigSource::TableBucketDefault); - assert_eq!(inherited.config.retain_recent_metadata_files, 3); - assert!(inherited.config.delete_enabled); - assert!(!inherited.config.background_enabled); - - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 1, - delete_enabled: false, - background_enabled: false, - ..Default::default() - }, - ) - .await - .expect("table maintenance override should persist"); - - let overridden = store - .get_effective_table_maintenance_config(bucket, "sales", "orders") - .await - .expect("effective maintenance override should load"); - - assert_eq!(overridden.source, TableMaintenanceConfigSource::TableOverride); - assert_eq!(overridden.config.retain_recent_metadata_files, 1); - assert!(!overridden.config.delete_enabled); - assert!(!overridden.config.background_enabled); - } - - #[tokio::test] - async fn maintenance_config_accepts_background_enabled_worker_runtime_controls() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - - let bucket_config = store - .put_table_bucket_maintenance_config( - bucket, - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 1, - delete_enabled: false, - background_enabled: true, - worker_paused: true, - worker_lease_timeout_seconds: 60, - ..Default::default() - }, - ) - .await - .expect("background maintenance bucket config should persist"); - assert!(bucket_config.background_enabled); - assert!(bucket_config.worker_paused); - assert_eq!(bucket_config.worker_lease_timeout_seconds, 60); - - let table_config = store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 1, - delete_enabled: false, - background_enabled: true, - worker_paused: false, - worker_lease_timeout_seconds: 120, - ..Default::default() - }, - ) - .await - .expect("background maintenance table config should persist"); - assert!(table_config.background_enabled); - assert!(!table_config.worker_paused); - assert_eq!(table_config.worker_lease_timeout_seconds, 120); - } - - #[tokio::test] - async fn maintenance_config_accepts_retry_and_quarantine_policy() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - - let config = store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 2, - delete_enabled: false, - background_enabled: false, - max_retry_attempts: 3, - retry_initial_backoff_seconds: 10, - retry_max_backoff_seconds: 60, - quarantine_enabled: true, - quarantine_retention_seconds: 86_400, - ..Default::default() - }, - ) - .await - .expect("retry and quarantine maintenance config should persist"); - - assert_eq!(config.max_retry_attempts, 3); - assert_eq!(config.retry_initial_backoff_seconds, 10); - assert_eq!(config.retry_max_backoff_seconds, 60); - assert!(config.quarantine_enabled); - assert_eq!(config.quarantine_retention_seconds, 86_400); - } - - #[tokio::test] - async fn maintenance_config_rejects_retry_backoff_above_limit() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - - let initial_err = store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 1, - delete_enabled: false, - background_enabled: false, - max_retry_attempts: 1, - retry_initial_backoff_seconds: TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS.saturating_add(1), - retry_max_backoff_seconds: TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS.saturating_add(1), - ..Default::default() - }, - ) - .await - .unwrap_err(); - assert_matches!(initial_err, TableCatalogStoreError::Invalid(_)); - - let max_err = store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 1, - delete_enabled: false, - background_enabled: false, - max_retry_attempts: 1, - retry_initial_backoff_seconds: TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS, - retry_max_backoff_seconds: TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS.saturating_add(1), - ..Default::default() - }, - ) - .await - .unwrap_err(); - assert_matches!(max_err, TableCatalogStoreError::Invalid(_)); - } - - #[tokio::test] - async fn maintenance_run_rejects_existing_invalid_retry_config_before_scheduling() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - let config_path = store - .paths - .table_maintenance_config_path(bucket, &namespace, &table, "table-id"); - store - .write_entry( - store.catalog_bucket(), - &config_path, - &TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 1, - delete_enabled: false, - background_enabled: false, - max_retry_attempts: 1, - retry_initial_backoff_seconds: u64::MAX, - retry_max_backoff_seconds: u64::MAX, - ..Default::default() - }, - TableCatalogPutPrecondition::Any, - ) - .await - .expect("invalid legacy maintenance config should be seeded"); - - let err = store - .run_table_metadata_maintenance(bucket, "sales", "orders", true, Some("worker-a".to_string())) - .await - .unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Invalid(_)); - } - - #[tokio::test] - async fn maintenance_run_persists_latest_job_alias_with_worker_and_lease_context() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - store - .put_table_bucket_maintenance_config( - bucket, - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 0, - delete_enabled: false, - background_enabled: false, - ..Default::default() - }, - ) - .await - .expect("bucket default maintenance config should persist"); - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - - let report = store - .run_table_metadata_maintenance(bucket, "sales", "orders", false, Some("worker-a".to_string())) - .await - .expect("metadata maintenance run should succeed"); - - assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Successful); - assert_eq!(report.job.config_source, TableMaintenanceConfigSource::TableBucketDefault); - assert_eq!(report.job.worker_id.as_deref(), Some("worker-a")); - assert!(!report.job.lease_id.is_empty()); - assert_eq!(report.job.attempt, 1); - assert_eq!(report.job.max_retry_attempts, 0); - assert!(report.job.next_retry_after.is_none()); - assert!(report.job.heartbeat_at.is_some()); - assert!(report.job.started_at.is_some()); - assert!(report.job.finished_at.is_some()); - - let latest = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", "latest") - .await - .expect("latest maintenance lookup should succeed") - .expect("latest maintenance job should be stored"); - let current_alias = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", "current") - .await - .expect("current maintenance lookup should succeed") - .expect("current maintenance job should be stored"); - - assert_eq!(latest.job.job_id, report.job.job_id); - assert_eq!(current_alias.job.job_id, report.job.job_id); - } - - #[tokio::test] - async fn maintenance_delete_request_records_failed_job_when_delete_is_disabled() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 0, - delete_enabled: false, - background_enabled: false, - max_retry_attempts: 2, - retry_initial_backoff_seconds: 10, - retry_max_backoff_seconds: 30, - quarantine_enabled: true, - quarantine_retention_seconds: 86_400, - ..Default::default() - }, - ) - .await - .expect("table maintenance override should persist"); - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - - let report = store - .run_table_metadata_maintenance(bucket, "sales", "orders", true, Some("worker-a".to_string())) - .await - .expect("disabled delete request should still persist a failed maintenance job"); - - assert_eq!(report.job.operation, TableMetadataMaintenanceOperation::Delete); - assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Failed); - assert_eq!(report.job.config_source, TableMaintenanceConfigSource::TableOverride); - assert_eq!(report.job.max_retry_attempts, 2); - assert!(report.job.next_retry_after.is_some()); - assert!(report.job.quarantine_enabled); - assert_eq!(report.job.quarantine_retention_seconds, 86_400); - assert!( - report - .job - .failure_reason - .as_deref() - .is_some_and(|reason| reason.contains("disabled")) - ); - assert_eq!( - report.job.recommended_actions, - vec![ - TableMaintenanceRecommendedAction::EnableDelete, - TableMaintenanceRecommendedAction::WaitForRetryBackoff, - ] - ); - assert!(backend.object_exists(bucket, &old).await.unwrap()); - - let latest = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", "latest") - .await - .expect("latest maintenance lookup should succeed") - .expect("failed maintenance job should be stored"); - assert_eq!(latest.job.job_id, report.job.job_id); - assert_eq!(latest.job.status, TableMetadataMaintenanceJobStatus::Failed); - assert_eq!(latest.job.recommended_actions, report.job.recommended_actions); - } - - #[tokio::test] - async fn maintenance_worker_run_skips_when_background_is_disabled() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let table_path = store.paths.table_entry_path(bucket, &namespace, &table); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - let before_worker_run = backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &table_path).await; - - let report = store - .run_table_metadata_maintenance_worker_once(bucket, "sales", "orders", "worker-a".to_string()) - .await - .expect("disabled background worker tick should report a safe no-op"); - - assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Disabled); - assert_eq!(report.job.worker_id.as_deref(), Some("worker-a")); - assert_eq!( - report.job.recommended_actions, - vec![TableMaintenanceRecommendedAction::EnableBackgroundMaintenance] - ); - assert_eq!(report.job.deleted_metadata_file_count, 0); - assert_eq!( - backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &table_path).await, - before_worker_run + 1 - ); - assert!(backend.object_exists(bucket, &old).await.unwrap()); - } - - #[tokio::test] - async fn maintenance_worker_run_honors_paused_config() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 0, - delete_enabled: true, - background_enabled: true, - worker_paused: true, - ..Default::default() - }, - ) - .await - .expect("paused background maintenance config should persist"); - - let report = store - .run_table_metadata_maintenance_worker_once(bucket, "sales", "orders", "worker-a".to_string()) - .await - .expect("paused worker tick should report a safe no-op"); - - assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Paused); - assert_eq!(report.job.operation, TableMetadataMaintenanceOperation::DryRun); - assert_eq!( - report.job.recommended_actions, - vec![TableMaintenanceRecommendedAction::ResumeMaintenanceWorker] - ); - assert_eq!(report.job.deleted_metadata_file_count, 0); - assert!(backend.object_exists(bucket, &old).await.unwrap()); - } - - #[tokio::test] - async fn maintenance_worker_run_claims_table_entry_write_lock() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let table_path = store.paths.table_entry_path(bucket, &namespace, &table); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - let before_worker_run = backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &table_path).await; - - let report = store - .run_table_metadata_maintenance_worker_once("analytics", "sales", "orders", "worker-a".to_string()) - .await - .expect("worker run should complete"); - - assert_eq!(report.job.worker_id.as_deref(), Some("worker-a")); - assert_eq!( - backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &table_path).await, - before_worker_run + 2 - ); - } - - #[tokio::test] - async fn maintenance_scheduler_report_marks_disabled_default() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - - let report = store - .get_table_maintenance_scheduler_report_at( - bucket, - "sales", - "orders", - OffsetDateTime::UNIX_EPOCH + Duration::seconds(100), - ) - .await - .expect("scheduler report should load"); - - assert_eq!(report.status, TableMaintenanceSchedulerStatus::Disabled); - assert_eq!(report.config_source, TableMaintenanceConfigSource::Default); - assert!(!report.background_enabled); - assert_eq!( - report.recommended_actions, - vec![TableMaintenanceRecommendedAction::EnableBackgroundMaintenance] - ); - assert!(report.current_job.is_none()); - assert!(report.audit_timeline.is_empty()); - assert!(!report.quarantine.active); - } - - #[tokio::test] - async fn maintenance_scheduler_run_queues_one_durable_job() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - retain_recent_metadata_files: 2, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - - let result = store - .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) - .await - .expect("scheduler tick should queue maintenance"); - - assert_eq!(result.report.job.status, TableMetadataMaintenanceJobStatus::Queued); - assert_eq!(result.report.job.scheduler_id.as_deref(), Some("scheduler-a")); - assert!(!result.report.job.scheduler_lease_id.is_empty()); - assert_eq!(result.report.job.retain_recent_metadata_files, 2); - assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::Queued); - assert_eq!( - result.scheduler.current_job.as_ref().map(|job| job.job_id.as_str()), - Some(result.report.job.job_id.as_str()) - ); - assert_eq!( - result.report.audit_events.last().map(|event| event.action.clone()), - Some(TableMaintenanceAuditAction::SchedulerQueued) - ); - } - - #[tokio::test] - async fn maintenance_scheduler_run_persists_disabled_control_report() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - - let result = store - .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) - .await - .expect("disabled scheduler tick should persist a control report"); - - assert_eq!(result.report.job.status, TableMetadataMaintenanceJobStatus::Disabled); - assert_eq!(result.report.job.scheduler_id.as_deref(), Some("scheduler-a")); - let stored = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", &result.report.job.job_id) - .await - .expect("scheduler control report lookup should succeed") - .expect("scheduler control report should be durable"); - assert_eq!(stored.job.job_id, result.report.job.job_id); - assert_eq!(stored.job.status, TableMetadataMaintenanceJobStatus::Disabled); - assert_eq!( - stored.audit_events.last().map(|event| event.action.clone()), - Some(TableMaintenanceAuditAction::SchedulerControl) - ); - let scheduler = store - .get_table_maintenance_scheduler_report_at(bucket, "sales", "orders", now) - .await - .expect("scheduler report should load"); - assert_eq!(scheduler.audit_timeline.len(), 1); - assert_eq!(scheduler.audit_timeline[0].job_id, result.report.job.job_id); - } - - #[tokio::test] - async fn maintenance_scheduler_run_reuses_active_queued_job() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - worker_lease_timeout_seconds: 300, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - - let first = store - .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) - .await - .expect("first scheduler tick should queue maintenance"); - let second = store - .run_table_maintenance_scheduler_once_at( - bucket, - "sales", - "orders", - "scheduler-b".to_string(), - now + Duration::seconds(30), - ) - .await - .expect("second scheduler tick should reuse the queued job"); - - assert_eq!(second.report.job.job_id, first.report.job.job_id); - assert_eq!(second.report.job.scheduler_id.as_deref(), Some("scheduler-a")); - assert_eq!(second.report.job.status, TableMetadataMaintenanceJobStatus::Queued); - assert_eq!(second.scheduler.audit_timeline.len(), 1); - } - - #[tokio::test] - async fn maintenance_worker_claims_queued_job_before_running_it() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - let queued = store - .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) - .await - .expect("scheduler tick should queue maintenance"); - - let finished = store - .run_table_metadata_maintenance_worker_once_at( - bucket, - "sales", - "orders", - "worker-a".to_string(), - now + Duration::seconds(10), - ) - .await - .expect("worker tick should claim and finish the queued job"); - - assert_eq!(finished.job.job_id, queued.report.job.job_id); - assert_eq!(finished.job.status, TableMetadataMaintenanceJobStatus::Successful); - assert_eq!(finished.job.worker_id.as_deref(), Some("worker-a")); - assert_eq!(finished.job.attempt, 1); - assert_eq!(finished.job.scheduler_id.as_deref(), Some("scheduler-a")); - let actions = finished - .audit_events - .iter() - .map(|event| event.action.clone()) - .collect::>(); - assert_eq!( - actions, - vec![ - TableMaintenanceAuditAction::Planned, - TableMaintenanceAuditAction::SchedulerQueued, - TableMaintenanceAuditAction::WorkerStarted, - TableMaintenanceAuditAction::WorkerSucceeded, - ] - ); - } - - #[tokio::test] - async fn maintenance_worker_preserves_queued_dry_run_after_delete_is_enabled() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let table_root = format!("{}{}/", default_table_root_prefix(&namespace), table.as_str()); - let manifest_list = format!("{metadata_dir}/snap-10.avro"); - let manifest = format!("{metadata_dir}/manifest-10.avro"); - let data_file = format!("{table_root}data/part-00001.parquet"); - let orphan_data = format!("{table_root}data/orphan.parquet"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) - .await; - backend.seed_object(bucket, &data_file, b"data".to_vec()).await; - backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "manifest-list": manifest_list - } - ], - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - delete_enabled: false, - ..Default::default() - }, - ) - .await - .expect("dry-run background maintenance config should persist"); - let queued = store - .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) - .await - .expect("scheduler tick should queue dry-run maintenance"); - assert_eq!(queued.report.job.operation, TableMetadataMaintenanceOperation::DryRun); - assert_eq!(queued.report.job.deletable_object_count, 1); - - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - delete_enabled: true, - ..Default::default() - }, - ) - .await - .expect("delete-enabled background maintenance config should persist"); - let finished = store - .run_table_metadata_maintenance_worker_once_at( - bucket, - "sales", - "orders", - "worker-a".to_string(), - now + Duration::seconds(10), - ) - .await - .expect("worker tick should preserve the queued dry-run operation"); - - assert_eq!(finished.job.job_id, queued.report.job.job_id); - assert_eq!(finished.job.operation, TableMetadataMaintenanceOperation::DryRun); - assert_eq!(finished.job.status, TableMetadataMaintenanceJobStatus::Successful); - assert_eq!(finished.job.deleted_object_count, 0); - assert!(backend.object_exists(bucket, &orphan_data).await.unwrap()); - } - - #[tokio::test] - async fn maintenance_scheduler_run_recovers_expired_queued_job() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1000); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - worker_lease_timeout_seconds: 60, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - let first = store - .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) - .await - .expect("first scheduler tick should queue maintenance"); - - let second = store - .run_table_maintenance_scheduler_once_at( - bucket, - "sales", - "orders", - "scheduler-b".to_string(), - now + Duration::seconds(120), - ) - .await - .expect("scheduler tick should recover expired queued maintenance"); - - assert_ne!(second.report.job.job_id, first.report.job.job_id); - let expired = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", &first.report.job.job_id) - .await - .expect("expired queued job lookup should succeed") - .expect("expired queued job should remain addressable"); - assert_eq!(expired.job.status, TableMetadataMaintenanceJobStatus::Failed); - assert!( - expired - .job - .failure_reason - .as_deref() - .is_some_and(|reason| reason.contains("scheduler lease expired")) - ); - assert_eq!( - expired.audit_events.last().map(|event| event.action.clone()), - Some(TableMaintenanceAuditAction::SchedulerLeaseExpired) - ); - assert_eq!(second.report.job.status, TableMetadataMaintenanceJobStatus::Queued); - assert_eq!(second.report.job.scheduler_id.as_deref(), Some("scheduler-b")); - } - - #[tokio::test] - async fn maintenance_scheduler_report_surfaces_active_backpressure_and_audit_timeline() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - worker_lease_timeout_seconds: 300, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - let mut running = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - running.job.status = TableMetadataMaintenanceJobStatus::Running; - running.job.worker_id = Some("worker-a".to_string()); - running.job.lease_id = "lease-a".to_string(); - running.job.heartbeat_at = Some(maintenance_timestamp(now - Duration::seconds(10))); - store - .put_table_metadata_maintenance_report(&running) - .await - .expect("running maintenance report should be seeded"); - - let report = store - .get_table_maintenance_scheduler_report_at(bucket, "sales", "orders", now) - .await - .expect("scheduler report should load"); - - assert_eq!(report.status, TableMaintenanceSchedulerStatus::Backpressured); - assert_eq!( - report.current_job.as_ref().map(|job| job.job_id.as_str()), - Some(running.job.job_id.as_str()) - ); - assert_eq!(report.recommended_actions, vec![TableMaintenanceRecommendedAction::WaitForActiveWorker]); - assert_eq!(report.audit_timeline.len(), 1); - assert_eq!(report.audit_timeline[0].job_id, running.job.job_id); - assert_eq!(report.audit_timeline[0].status, TableMetadataMaintenanceJobStatus::Running); - assert_eq!(report.audit_timeline[0].worker_id.as_deref(), Some("worker-a")); - } - - #[tokio::test] - async fn maintenance_scheduler_report_surfaces_quarantine_boundary() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - quarantine_enabled: true, - quarantine_retention_seconds: 86_400, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - let mut failed = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - failed.job.status = TableMetadataMaintenanceJobStatus::Failed; - failed.job.failure_reason = Some("quarantine retained failed cleanup candidates".to_string()); - failed.job.quarantine_enabled = true; - failed.job.quarantine_retention_seconds = 86_400; - failed.job.quarantined_object_count = 2; - failed.job.finished_at = Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH + Duration::seconds(90))); - store - .put_table_metadata_maintenance_report(&failed) - .await - .expect("failed maintenance report should be seeded"); - - let report = store - .get_table_maintenance_scheduler_report_at( - bucket, - "sales", - "orders", - OffsetDateTime::UNIX_EPOCH + Duration::seconds(100), - ) - .await - .expect("scheduler report should load"); - - assert_eq!(report.status, TableMaintenanceSchedulerStatus::Quarantined); - assert!(report.quarantine.active); - assert_eq!(report.quarantine.retention_seconds, 86_400); - assert_eq!(report.quarantine.quarantined_object_count, 2); - assert_eq!(report.quarantine.source_job_id.as_deref(), Some(failed.job.job_id.as_str())); - assert!( - report - .recommended_actions - .contains(&TableMaintenanceRecommendedAction::ReviewQuarantine) - ); - } - - #[tokio::test] - async fn maintenance_quarantine_retry_clears_boundary_and_unblocks_scheduler() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let now = OffsetDateTime::now_utc(); - let (_namespace, _table, failed) = - seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; - - let result = store - .apply_table_maintenance_quarantine_operation( - bucket, - "sales", - "orders", - &failed.job.job_id, - TableMaintenanceQuarantineOperationRequest { - action: TableMaintenanceQuarantineAction::Retry, - reason: Some("operator reviewed retained candidates".to_string()), - }, - ) - .await - .expect("quarantine retry should update the current maintenance job"); - - assert_eq!(result.action, TableMaintenanceQuarantineAction::Retry); - assert_eq!(result.report.job.job_id, failed.job.job_id); - assert_eq!(result.report.job.quarantined_object_count, 0); - assert!(result.report.job.next_retry_after.is_none()); - let event = result - .report - .audit_events - .last() - .expect("quarantine retry should append an audit event"); - assert_eq!(event.action, TableMaintenanceAuditAction::QuarantineRetry); - assert_eq!(event.actor, TableMaintenanceAuditActor::Operator); - assert_eq!(event.reason.as_deref(), Some("operator reviewed retained candidates")); - assert_eq!(event.before_status, Some(TableMetadataMaintenanceJobStatus::Failed)); - assert_eq!(event.after_status, Some(TableMetadataMaintenanceJobStatus::Failed)); - assert_eq!(event.before_quarantined_object_count, Some(2)); - assert_eq!(event.after_quarantined_object_count, Some(0)); - assert!( - !result - .report - .job - .recommended_actions - .contains(&TableMaintenanceRecommendedAction::ReviewQuarantine) - ); - assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::Ready); - assert!(!result.scheduler.quarantine.active); - let summary = result - .scheduler - .audit_timeline - .iter() - .find(|summary| summary.job_id == failed.job.job_id) - .expect("scheduler timeline should include the retried job"); - assert_eq!( - summary.audit_events.last().map(|event| event.action.clone()), - Some(TableMaintenanceAuditAction::QuarantineRetry) - ); - - let current_report = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", MAINTENANCE_JOB_ALIAS_CURRENT) - .await - .expect("current maintenance report should load") - .expect("current maintenance report should exist"); - assert_eq!(current_report.job.job_id, failed.job.job_id); - assert_eq!(current_report.job.quarantined_object_count, 0); - } - - #[tokio::test] - async fn maintenance_quarantine_inspect_reports_without_mutating_current_job() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let now = OffsetDateTime::now_utc(); - let (_namespace, _table, failed) = - seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; - - let result = store - .apply_table_maintenance_quarantine_operation( - bucket, - "sales", - "orders", - &failed.job.job_id, - TableMaintenanceQuarantineOperationRequest { - action: TableMaintenanceQuarantineAction::Inspect, - reason: Some("ignored for inspect".to_string()), - }, - ) - .await - .expect("quarantine inspect should load the maintenance job"); - - assert_eq!(result.action, TableMaintenanceQuarantineAction::Inspect); - assert_eq!(result.report.job.job_id, failed.job.job_id); - assert_eq!(result.report.job.quarantined_object_count, 2); - let expected_retry_after = maintenance_timestamp(now + Duration::seconds(300)); - assert_eq!(result.report.job.next_retry_after.as_deref(), Some(expected_retry_after.as_str())); - assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::RetryDeferred); - - let current_report = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", MAINTENANCE_JOB_ALIAS_CURRENT) - .await - .expect("current maintenance report should load") - .expect("current maintenance report should exist"); - assert_eq!(current_report.job.quarantined_object_count, 2); - assert_eq!(current_report.audit_events, failed.audit_events); - } - - #[tokio::test] - async fn maintenance_quarantine_release_preserves_retry_deferral() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let now = OffsetDateTime::now_utc(); - let (_namespace, _table, failed) = - seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; - - let result = store - .apply_table_maintenance_quarantine_operation( - bucket, - "sales", - "orders", - &failed.job.job_id, - TableMaintenanceQuarantineOperationRequest { - action: TableMaintenanceQuarantineAction::Release, - reason: Some("objects retained for later retry".to_string()), - }, - ) - .await - .expect("quarantine release should update the current maintenance job"); - - assert_eq!(result.action, TableMaintenanceQuarantineAction::Release); - assert_eq!(result.report.job.quarantined_object_count, 0); - let expected_retry_after = maintenance_timestamp(now + Duration::seconds(300)); - assert_eq!(result.report.job.next_retry_after.as_deref(), Some(expected_retry_after.as_str())); - assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::RetryDeferred); - assert!(result.report.job.failure_reason.as_deref().is_some_and(|reason| { - reason.contains("maintenance quarantine released by operator") && reason.contains("objects retained for later retry") - })); - } - - #[tokio::test] - async fn maintenance_quarantine_abandon_clears_boundary_and_retry() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let now = OffsetDateTime::now_utc(); - let (_namespace, _table, failed) = - seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; - - let result = store - .apply_table_maintenance_quarantine_operation( - bucket, - "sales", - "orders", - &failed.job.job_id, - TableMaintenanceQuarantineOperationRequest { - action: TableMaintenanceQuarantineAction::Abandon, - reason: Some("operator accepted retained objects".to_string()), - }, - ) - .await - .expect("quarantine abandon should update the current maintenance job"); - - assert_eq!(result.action, TableMaintenanceQuarantineAction::Abandon); - assert_eq!(result.report.job.quarantined_object_count, 0); - assert!(result.report.job.next_retry_after.is_none()); - assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::Ready); - assert!(result.report.job.failure_reason.as_deref().is_some_and(|reason| { - reason.contains("maintenance quarantine abandoned by operator") - && reason.contains("operator accepted retained objects") - })); - } - - #[tokio::test] - async fn maintenance_quarantine_rejects_mutating_non_current_job() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let now = OffsetDateTime::now_utc(); - let (_namespace, _table, old_failed) = - seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; - let mut current_failed = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - current_failed.job.status = TableMetadataMaintenanceJobStatus::Failed; - current_failed.job.quarantine_enabled = true; - current_failed.job.quarantine_retention_seconds = 86_400; - current_failed.job.quarantined_object_count = 1; - store - .put_table_metadata_maintenance_report(¤t_failed) - .await - .expect("new current maintenance report should be seeded"); - - let error = store - .apply_table_maintenance_quarantine_operation( - bucket, - "sales", - "orders", - &old_failed.job.job_id, - TableMaintenanceQuarantineOperationRequest { - action: TableMaintenanceQuarantineAction::Release, - reason: None, - }, - ) - .await - .expect_err("mutating a non-current quarantine job should fail"); - - assert_eq!(error, TableCatalogStoreError::Conflict("maintenance job is not current".to_string())); - } - - #[tokio::test] - async fn maintenance_worker_run_records_audit_timeline_events() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - background_enabled: true, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - - let report = store - .run_table_metadata_maintenance_worker_once(bucket, "sales", "orders", "worker-a".to_string()) - .await - .expect("maintenance worker should finish"); - - let actions = report - .audit_events - .iter() - .map(|event| event.action.clone()) - .collect::>(); - assert_eq!( - actions, - vec![ - TableMaintenanceAuditAction::Planned, - TableMaintenanceAuditAction::WorkerStarted, - TableMaintenanceAuditAction::WorkerSucceeded, - ] - ); - assert_eq!(report.audit_events[1].actor, TableMaintenanceAuditActor::Worker); - assert_eq!(report.audit_events[1].before_status, Some(TableMetadataMaintenanceJobStatus::Successful)); - assert_eq!(report.audit_events[2].after_status, Some(TableMetadataMaintenanceJobStatus::Successful)); - - let scheduler = store - .get_table_maintenance_scheduler_report(bucket, "sales", "orders") - .await - .expect("scheduler report should load"); - let summary = scheduler.current_job.expect("current job should be visible"); - assert_eq!(summary.job_id, report.job.job_id); - assert_eq!(summary.audit_events, report.audit_events); - } - - #[tokio::test] - async fn maintenance_heartbeat_appends_worker_audit_event() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - let mut running = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - running.job.status = TableMetadataMaintenanceJobStatus::Running; - running.job.worker_id = Some("worker-a".to_string()); - running.job.lease_id = "lease-a".to_string(); - running.job.heartbeat_at = Some(maintenance_timestamp(now - Duration::seconds(10))); - store - .put_table_metadata_maintenance_report(&running) - .await - .expect("running maintenance report should be seeded"); - - let heartbeat = store - .heartbeat_table_metadata_maintenance_job_at( - TableMaintenanceHeartbeatRef { - table_bucket: bucket, - namespace: "sales", - table: "orders", - job_id: &running.job.job_id, - lease_id: "lease-a", - worker_id: "worker-a", - }, - now, - ) - .await - .expect("heartbeat should update the running job"); - - let event = heartbeat.audit_events.last().expect("heartbeat should append an audit event"); - assert_eq!(event.action, TableMaintenanceAuditAction::WorkerHeartbeat); - assert_eq!(event.actor, TableMaintenanceAuditActor::Worker); - assert_eq!(event.before_status, Some(TableMetadataMaintenanceJobStatus::Running)); - assert_eq!(event.after_status, Some(TableMetadataMaintenanceJobStatus::Running)); - } - - #[tokio::test] - async fn maintenance_worker_run_defers_until_retry_after() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 0, - delete_enabled: false, - background_enabled: true, - max_retry_attempts: 2, - retry_initial_backoff_seconds: 60, - retry_max_backoff_seconds: 60, - ..Default::default() - }, - ) - .await - .expect("retry-enabled maintenance config should persist"); - let mut failed = store - .run_table_metadata_maintenance(bucket, "sales", "orders", true, Some("worker-a".to_string())) - .await - .expect("delete failure should be recorded when delete is disabled"); - failed.job.next_retry_after = Some(maintenance_timestamp(now + Duration::seconds(30))); - store - .put_table_metadata_maintenance_report(&failed) - .await - .expect("failed retry report should be seeded"); - - let deferred = store - .run_table_metadata_maintenance_worker_once_at(bucket, "sales", "orders", "worker-b".to_string(), now) - .await - .expect("worker tick should defer while retry backoff is active"); - - assert_eq!(deferred.job.job_id, failed.job.job_id); - assert_eq!(deferred.job.status, TableMetadataMaintenanceJobStatus::Failed); - assert_eq!(deferred.job.worker_id.as_deref(), Some("worker-a")); - assert!( - deferred - .job - .recommended_actions - .contains(&TableMaintenanceRecommendedAction::WaitForRetryBackoff) - ); - assert!(backend.object_exists(bucket, &old).await.unwrap()); - } - - #[tokio::test] - async fn maintenance_worker_run_backpressures_active_running_job() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 0, - delete_enabled: false, - background_enabled: true, - worker_lease_timeout_seconds: 300, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - let mut running = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - running.job.status = TableMetadataMaintenanceJobStatus::Running; - running.job.worker_id = Some("worker-a".to_string()); - running.job.lease_id = "lease-a".to_string(); - running.job.heartbeat_at = Some(maintenance_timestamp(now - Duration::seconds(10))); - store - .put_table_metadata_maintenance_report(&running) - .await - .expect("running maintenance report should be seeded"); - - let report = store - .run_table_metadata_maintenance_worker_once_at(bucket, "sales", "orders", "worker-b".to_string(), now) - .await - .expect("worker tick should return the active running job"); - - assert_eq!(report.job.job_id, running.job.job_id); - assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Running); - assert_eq!(report.job.worker_id.as_deref(), Some("worker-a")); - assert_eq!( - report.job.recommended_actions, - vec![TableMaintenanceRecommendedAction::WaitForActiveWorker] - ); - } - - #[tokio::test] - async fn maintenance_worker_run_recovers_expired_running_job() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1000); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - store - .put_table_maintenance_config( - bucket, - "sales", - "orders", - TableMaintenanceConfig { - version: TABLE_MAINTENANCE_CONFIG_VERSION, - retain_recent_metadata_files: 0, - delete_enabled: false, - background_enabled: true, - worker_lease_timeout_seconds: 60, - ..Default::default() - }, - ) - .await - .expect("background maintenance config should persist"); - let mut running = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - let expired_job_id = running.job.job_id.clone(); - running.job.status = TableMetadataMaintenanceJobStatus::Running; - running.job.worker_id = Some("worker-a".to_string()); - running.job.lease_id = "lease-a".to_string(); - running.job.heartbeat_at = Some(maintenance_timestamp(now - Duration::seconds(120))); - store - .put_table_metadata_maintenance_report(&running) - .await - .expect("expired running maintenance report should be seeded"); - - let report = store - .run_table_metadata_maintenance_worker_once_at(bucket, "sales", "orders", "worker-b".to_string(), now) - .await - .expect("worker tick should recover expired running job and run again"); - - assert_ne!(report.job.job_id, expired_job_id); - assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Successful); - assert_eq!(report.job.worker_id.as_deref(), Some("worker-b")); - - let expired = store - .get_table_metadata_maintenance_report(bucket, "sales", "orders", &expired_job_id) - .await - .expect("expired job lookup should succeed") - .expect("expired job should remain addressable"); - assert_eq!(expired.job.status, TableMetadataMaintenanceJobStatus::Failed); - assert!( - expired - .job - .failure_reason - .as_deref() - .is_some_and(|reason| reason.contains("lease expired")) - ); - assert_eq!( - expired.job.recommended_actions, - vec![TableMaintenanceRecommendedAction::InvestigateFailure] - ); - let event = expired - .audit_events - .last() - .expect("expired lease recovery should append an audit event"); - assert_eq!(event.action, TableMaintenanceAuditAction::WorkerLeaseExpired); - assert_eq!(event.actor, TableMaintenanceAuditActor::Scheduler); - assert_eq!(event.before_status, Some(TableMetadataMaintenanceJobStatus::Running)); - assert_eq!(event.after_status, Some(TableMetadataMaintenanceJobStatus::Failed)); - } - - #[tokio::test] - async fn maintenance_worker_heartbeat_updates_current_running_job() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let first = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); - let second = OffsetDateTime::UNIX_EPOCH + Duration::seconds(130); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; - backend - .seed_object( - bucket, - &default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"), - br#"{"metadata-log":[]}"#.to_vec(), - ) - .await; - let mut running = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("maintenance report should be planned"); - running.job.status = TableMetadataMaintenanceJobStatus::Running; - running.job.worker_id = Some("worker-a".to_string()); - running.job.lease_id = "lease-a".to_string(); - running.job.heartbeat_at = Some(maintenance_timestamp(first)); - let job_id = running.job.job_id.clone(); - store - .put_table_metadata_maintenance_report(&running) - .await - .expect("running maintenance report should be seeded"); - - let heartbeat = store - .heartbeat_table_metadata_maintenance_job_at( - TableMaintenanceHeartbeatRef { - table_bucket: bucket, - namespace: "sales", - table: "orders", - job_id: &job_id, - lease_id: "lease-a", - worker_id: "worker-a", - }, - second, - ) - .await - .expect("heartbeat should update the current running job"); - - assert_eq!(heartbeat.job.job_id, job_id); - assert_eq!(heartbeat.job.status, TableMetadataMaintenanceJobStatus::Running); - assert_eq!(heartbeat.job.heartbeat_at.as_deref(), Some(maintenance_timestamp(second).as_str())); - } - - #[tokio::test] - async fn maintenance_reachability_reports_manifest_lists_as_manual_review() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let manifest_list = format!("{}/snap-10.avro", default_table_metadata_dir_path(&namespace, &table)); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend.seed_object(bucket, &manifest_list, b"avro".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [], - "schemas": [], - "partition-specs": [], - "sort-orders": [], - "snapshots": [ - { - "snapshot-id": 10, - "manifest-list": manifest_list - } - ], - "snapshot-log": [ - { - "timestamp-ms": 1, - "snapshot-id": 10 - } - ], - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("metadata maintenance dry-run should succeed"); - - assert_eq!(report.cleanup_candidate_locations, vec![old]); - let manifest_report = report - .referenced_object_reports - .iter() - .find(|object| object.object_location == manifest_list) - .expect("manifest list should be reported as a referenced object"); - assert_eq!(manifest_report.object_kind, TableMetadataMaintenanceObjectKind::ManifestList); - assert_eq!(manifest_report.state, TableMetadataMaintenanceObjectState::ManualReviewRequired); - assert_eq!( - manifest_report.reasons, - vec![ - TableMetadataMaintenanceReason::ManifestList, - TableMetadataMaintenanceReason::UnsupportedManifestAvro, - ] - ); - assert_eq!( - report.reachability_graph.status, - TableMaintenanceReachabilityGraphStatus::ManualReviewRequired - ); - assert_eq!(report.reachability_graph.metadata_file_count, 2); - assert_eq!(report.reachability_graph.manifest_list_count, 1); - assert_eq!(report.reachability_graph.manual_review_count, 1); - assert!( - report - .reachability_graph - .reasons - .contains(&TableMaintenanceReachabilityGraphReason::ManifestAvroReaderUnavailable) - ); - } - - #[tokio::test] - async fn maintenance_reachability_expands_manifest_avro_references() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let table_root = format!("{}{}/", default_table_root_prefix(&namespace), table.as_str()); - let manifest_list = format!("{metadata_dir}/snap-10.avro"); - let manifest = format!("{metadata_dir}/manifest-10.avro"); - let data_file = format!("{table_root}data/part-00001.parquet"); - let delete_file = format!("{table_root}delete/pos-00001.parquet"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0), (&delete_file, 1)])) - .await; - backend.seed_object(bucket, &data_file, b"data".to_vec()).await; - backend.seed_object(bucket, &delete_file, b"delete".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "manifest-list": manifest_list - } - ], - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("metadata maintenance dry-run should succeed"); - - assert_eq!(report.reachability_graph.status, TableMaintenanceReachabilityGraphStatus::Complete); - assert_eq!(report.reachability_graph.manifest_list_count, 1); - assert_eq!(report.reachability_graph.manifest_file_count, 1); - assert_eq!(report.reachability_graph.data_file_count, 1); - assert_eq!(report.reachability_graph.delete_file_count, 1); - assert_eq!(report.reachability_graph.manual_review_count, 0); - assert!( - !report - .reachability_graph - .reasons - .contains(&TableMaintenanceReachabilityGraphReason::ManifestAvroReaderUnavailable) - ); - for (location, kind) in [ - (&manifest_list, TableMetadataMaintenanceObjectKind::ManifestList), - (&manifest, TableMetadataMaintenanceObjectKind::ManifestFile), - (&data_file, TableMetadataMaintenanceObjectKind::DataFile), - (&delete_file, TableMetadataMaintenanceObjectKind::DeleteFile), - ] { - let referenced = report - .referenced_object_reports - .iter() - .find(|object| object.object_location == *location) - .expect("referenced object should be reported"); - assert_eq!(referenced.object_kind, kind); - assert_eq!(referenced.state, TableMetadataMaintenanceObjectState::Retained); - } - assert!(report.cleanup_object_candidate_locations.is_empty()); - assert!(report.deletable_object_locations.is_empty()); - } - - #[tokio::test] - async fn maintenance_reachability_treats_v1_snapshot_manifests_as_reachable() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let data_dir = default_table_data_dir_path(&namespace, &table); - let manifest = format!("{metadata_dir}/manifest-10.avro"); - let data_file = format!("{data_dir}/part-00001.parquet"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) - .await; - backend.seed_object(bucket, &data_file, b"data".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "manifests": [manifest] - } - ], - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("metadata maintenance dry-run should succeed"); - - assert_eq!(report.reachability_graph.status, TableMaintenanceReachabilityGraphStatus::Complete); - assert_eq!(report.reachability_graph.manifest_file_count, 1); - assert_eq!(report.reachability_graph.data_file_count, 1); - assert!(report.cleanup_object_candidate_locations.is_empty()); - assert!(report.deletable_object_locations.is_empty()); - for location in [&manifest, &data_file] { - let referenced = report - .referenced_object_reports - .iter() - .find(|object| object.object_location == *location) - .expect("v1 manifest reference should be retained"); - assert_eq!(referenced.state, TableMetadataMaintenanceObjectState::Retained); - } - } - - #[tokio::test] - async fn maintenance_reachability_uses_table_warehouse_object_paths() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let manifest_list = "tables/table-id/metadata/snap-10.avro".to_string(); - let manifest = "tables/table-id/metadata/manifest-10.avro".to_string(); - let data_file = "tables/table-id/data/part-00001.parquet".to_string(); - let orphan_data = "tables/table-id/data/orphan.parquet".to_string(); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) - .await; - backend.seed_object(bucket, &data_file, b"data".to_vec()).await; - backend.seed_object(bucket, &orphan_data, b"orphan".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "manifest-list": format!("s3://{bucket}/{manifest_list}") - } - ], - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("metadata maintenance dry-run should succeed"); - - assert_eq!(report.reachability_graph.status, TableMaintenanceReachabilityGraphStatus::Complete); - assert!(report.referenced_object_reports.iter().any( - |object| object.object_location == manifest_list && object.state == TableMetadataMaintenanceObjectState::Retained - )); - assert!( - report.referenced_object_reports.iter().any( - |object| object.object_location == data_file && object.state == TableMetadataMaintenanceObjectState::Retained - ) - ); - assert_eq!(report.cleanup_object_candidate_locations, vec![orphan_data.clone()]); - assert_eq!(report.deletable_object_locations, vec![orphan_data]); - } - - #[tokio::test] - async fn maintenance_reachability_fails_closed_when_retained_metadata_is_unreadable() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let orphan_data = format!("{}/orphan.parquet", default_table_data_dir_path(&namespace, &table)); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"not-json".to_vec()).await; - backend.seed_object(bucket, &orphan_data, b"orphan".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [ - { - "timestamp-ms": 1, - "metadata-file": old - } - ], - "snapshots": [] - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("metadata maintenance dry-run should succeed"); - - assert_eq!( - report.reachability_graph.status, - TableMaintenanceReachabilityGraphStatus::ManualReviewRequired - ); - assert!(report.cleanup_object_candidate_locations.is_empty()); - assert!(report.deletable_object_locations.is_empty()); - let retained_metadata = report - .referenced_object_reports - .iter() - .find(|object| object.object_location == old) - .expect("unreadable retained metadata should be reported"); - assert_eq!(retained_metadata.object_kind, TableMetadataMaintenanceObjectKind::MetadataFile); - assert_eq!(retained_metadata.state, TableMetadataMaintenanceObjectState::ManualReviewRequired); - assert!( - retained_metadata - .reasons - .contains(&TableMetadataMaintenanceReason::UnreadableMetadata) - ); - } - - #[tokio::test] - async fn maintenance_dry_run_reports_unreachable_manifest_data_and_delete_candidates() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let table_root = format!("{}{}/", default_table_root_prefix(&namespace), table.as_str()); - let manifest_list = format!("{metadata_dir}/snap-10.avro"); - let manifest = format!("{metadata_dir}/manifest-10.avro"); - let data_file = format!("{table_root}data/part-00001.parquet"); - let orphan_manifest = format!("{metadata_dir}/manifest-orphan.avro"); - let orphan_data = format!("{table_root}data/orphan.parquet"); - let orphan_delete = format!("{table_root}delete/orphan-delete.parquet"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) - .await; - backend.seed_object(bucket, &data_file, b"data".to_vec()).await; - backend.seed_object(bucket, &orphan_manifest, manifest_avro_bytes(&[])).await; - backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; - backend.seed_object(bucket, &orphan_delete, b"orphan-delete".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "manifest-list": manifest_list - } - ], - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .expect("metadata maintenance dry-run should succeed"); - let candidates = report - .cleanup_object_candidate_locations - .iter() - .cloned() - .collect::>(); - let deletable = report.deletable_object_locations.iter().cloned().collect::>(); - let expected = BTreeSet::from([orphan_data.clone(), orphan_delete.clone(), orphan_manifest.clone()]); - - assert_eq!(candidates, expected); - assert_eq!(deletable, expected); - assert_eq!( - object_cleanup_report(&report, &orphan_manifest).object_kind, - TableMetadataMaintenanceObjectKind::ManifestFile - ); - assert_eq!( - object_cleanup_report(&report, &orphan_data).object_kind, - TableMetadataMaintenanceObjectKind::DataFile - ); - assert_eq!( - object_cleanup_report(&report, &orphan_delete).object_kind, - TableMetadataMaintenanceObjectKind::DeleteFile - ); - } - - #[tokio::test] - async fn maintenance_delete_removes_only_planned_unreachable_table_objects() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let table_root = format!("{}{}/", default_table_root_prefix(&namespace), table.as_str()); - let manifest_list = format!("{metadata_dir}/snap-10.avro"); - let manifest = format!("{metadata_dir}/manifest-10.avro"); - let data_file = format!("{table_root}data/part-00001.parquet"); - let orphan_manifest = format!("{metadata_dir}/manifest-orphan.avro"); - let orphan_data = format!("{table_root}data/orphan.parquet"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) - .await; - backend.seed_object(bucket, &data_file, b"data".to_vec()).await; - backend.seed_object(bucket, &orphan_manifest, manifest_avro_bytes(&[])).await; - backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "manifest-list": manifest_list - } - ], - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let deleted = store - .delete_table_metadata_maintenance_candidates(bucket, "sales", "orders", 0) - .await - .expect("maintenance delete should succeed"); - - assert_eq!( - deleted.deletable_object_locations.iter().cloned().collect::>(), - BTreeSet::from([orphan_data.clone(), orphan_manifest.clone()]) - ); - assert!(backend.object_exists(bucket, &manifest_list).await.unwrap()); - assert!(backend.object_exists(bucket, &manifest).await.unwrap()); - assert!(backend.object_exists(bucket, &data_file).await.unwrap()); - assert!(!backend.object_exists(bucket, &orphan_manifest).await.unwrap()); - assert!(!backend.object_exists(bucket, &orphan_data).await.unwrap()); - } - - #[tokio::test] - async fn maintenance_dry_run_keeps_metadata_log_references() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let v1 = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let logged = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let v3 = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - for metadata in [&v1, &logged, &v3] { - backend.seed_object(bucket, metadata, b"{}".to_vec()).await; - } - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [ - { - "timestamp-ms": 1, - "metadata-file": logged - } - ] - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .unwrap(); - - assert!(report.retained_metadata_locations.contains(¤t)); - assert!(report.retained_metadata_locations.contains(&logged)); - assert_eq!(report.cleanup_candidate_locations, vec![v1, v3]); - } - - #[tokio::test] - async fn maintenance_dry_run_keeps_metadata_for_protected_snapshot_refs() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let orphan = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let tagged = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let unreferenced = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &orphan, b"{}".to_vec()).await; - backend - .seed_object( - bucket, - &tagged, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 10 - })) - .unwrap(), - ) - .await; - backend - .seed_object( - bucket, - &unreferenced, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 20 - })) - .unwrap(), - ) - .await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 30, - "metadata-log": [], - "refs": { - "main": { - "snapshot-id": 30, - "type": "branch" - }, - "audit": { - "snapshot-id": 10, - "type": "tag" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .unwrap(); - - assert!(report.retained_metadata_locations.contains(&tagged)); - assert_eq!(report.cleanup_candidate_locations, vec![orphan, unreferenced]); - } - - #[tokio::test] - async fn snapshot_expiration_plan_retains_current_recent_and_protected_refs() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 30, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "timestamp-ms": 1000, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-10.avro" - }, - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" - }, - { - "snapshot-id": 30, - "timestamp-ms": 3000, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-30.avro" - } - ], - "refs": { - "main": { - "snapshot-id": 30, - "type": "branch" - }, - "audit": { - "snapshot-id": 10, - "type": "tag" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_snapshot_expiration( - bucket, - "sales", - "orders", - TableSnapshotExpirationConfig { - min_snapshots_to_keep: 1, - max_snapshot_age_ms: 1, - }, - ) - .await - .expect("snapshot expiration planning should succeed"); - - let current_snapshot = snapshot_expiration_report(&report, 30); - assert_eq!(current_snapshot.state, TableSnapshotExpirationSnapshotState::Retained); - assert!( - current_snapshot - .reasons - .contains(&TableSnapshotExpirationReason::CurrentSnapshot) - ); - - let protected_snapshot = snapshot_expiration_report(&report, 10); - assert_eq!(protected_snapshot.state, TableSnapshotExpirationSnapshotState::ManualReviewRequired); - assert!( - protected_snapshot - .reasons - .contains(&TableSnapshotExpirationReason::ProtectedSnapshotRef) - ); - assert!( - protected_snapshot - .reasons - .contains(&TableSnapshotExpirationReason::UserDefinedSnapshotRef) - ); - - let expired_snapshot = snapshot_expiration_report(&report, 20); - assert_eq!(expired_snapshot.state, TableSnapshotExpirationSnapshotState::ExpirationCandidate); - assert!( - expired_snapshot - .reasons - .contains(&TableSnapshotExpirationReason::SnapshotAgeExpired) - ); - assert_eq!(report.expiration_candidate_count, 1); - assert_eq!(report.manual_review_count, 1); - } - - #[tokio::test] - async fn snapshot_expiration_plan_fails_closed_for_table_retention_property_conflicts() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 20, - "metadata-log": [], - "properties": { - "history.expire.min-snapshots-to-keep": "5" - }, - "snapshots": [ - { - "snapshot-id": 10, - "timestamp-ms": 1000, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-10.avro" - }, - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" - } - ], - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_snapshot_expiration( - bucket, - "sales", - "orders", - TableSnapshotExpirationConfig { - min_snapshots_to_keep: 1, - max_snapshot_age_ms: 1, - }, - ) - .await - .expect("snapshot expiration planning should succeed"); - - assert_eq!(report.expiration_candidate_count, 0); - assert_eq!(report.manual_review_count, 2); - for snapshot in &report.snapshot_reports { - assert_eq!(snapshot.state, TableSnapshotExpirationSnapshotState::ManualReviewRequired); - assert!( - snapshot - .reasons - .contains(&TableSnapshotExpirationReason::TableRetentionPropertyConflict) - ); - } - } - - #[tokio::test] - async fn snapshot_expiration_plan_requires_snapshot_timestamps() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 20, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 10, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-10.avro" - }, - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" - } - ], - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_snapshot_expiration( - bucket, - "sales", - "orders", - TableSnapshotExpirationConfig { - min_snapshots_to_keep: 1, - max_snapshot_age_ms: 1, - }, - ) - .await - .expect("snapshot expiration planning should succeed"); - - let missing_timestamp = snapshot_expiration_report(&report, 10); - assert_eq!(missing_timestamp.state, TableSnapshotExpirationSnapshotState::ManualReviewRequired); - assert!( - missing_timestamp - .reasons - .contains(&TableSnapshotExpirationReason::MissingSnapshotTimestamp) - ); - assert_eq!(report.expiration_candidate_count, 0); - assert_eq!(report.manual_review_count, 1); - } - - #[tokio::test] - async fn compaction_plan_reports_manifest_reader_gap_without_rewrite_candidates() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 20, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" - } - ], - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_compaction( - bucket, - "sales", - "orders", - TableCompactionPlanningConfig { - target_file_size_bytes: 512 * 1024 * 1024, - small_file_threshold_bytes: 64 * 1024 * 1024, - min_input_files: 2, - max_rewrite_bytes_per_job: 1024 * 1024 * 1024, - }, - ) - .await - .expect("compaction planning should succeed"); - - assert_eq!(report.status, TableCompactionPlanningStatus::ManualReviewRequired); - assert_eq!(report.candidate_file_count, 0); - assert_eq!(report.rewrite_group_count, 0); - assert_eq!(report.manual_review_count, 1); - let snapshot = compaction_snapshot_report(&report, 20); - assert_eq!(snapshot.status, TableCompactionPlanningStatus::ManualReviewRequired); - assert!(snapshot.reasons.contains(&TableCompactionPlanningReason::ManifestList)); - assert!( - snapshot - .reasons - .contains(&TableCompactionPlanningReason::ManifestAvroReaderUnavailable) - ); - } - - #[tokio::test] - async fn compaction_plan_reports_row_level_delete_files_without_rewrite_candidates() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let data_dir = default_table_data_dir_path(&namespace, &table); - let delete_dir = default_table_delete_dir_path(&namespace, &table); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - let manifest_list = format!("{metadata_dir}/snap-20.avro"); - let manifest = format!("{metadata_dir}/manifest-20.avro"); - let data_file = format!("{data_dir}/part-left.parquet"); - let position_delete_file = format!("{delete_dir}/pos-left.parquet"); - let equality_delete_file = format!("{delete_dir}/eq-left.parquet"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes(&[(&data_file, 0), (&position_delete_file, 1), (&equality_delete_file, 2)]), - ) - .await; - backend.seed_object(bucket, &data_file, parquet_i32_bytes(&[1, 2])).await; - backend - .seed_object(bucket, &position_delete_file, b"position-delete".to_vec()) - .await; - backend - .seed_object(bucket, &equality_delete_file, b"equality-delete".to_vec()) - .await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 20, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": manifest_list - } - ], - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - })) - .expect("current metadata should serialize"), - ) - .await; - - let config = TableCompactionPlanningConfig { - target_file_size_bytes: 512 * 1024 * 1024, - small_file_threshold_bytes: 64 * 1024 * 1024, - min_input_files: 2, - max_rewrite_bytes_per_job: 1024 * 1024 * 1024, - }; - let report = store - .plan_table_compaction(bucket, "sales", "orders", config.clone()) - .await - .expect("compaction planning should succeed"); - - assert_eq!(report.status, TableCompactionPlanningStatus::ManualReviewRequired); - assert_eq!(report.candidate_file_count, 1); - assert_eq!(report.rewrite_group_count, 0); - assert_eq!(report.manual_review_count, 1); - assert_eq!( - report.row_level_planning.status, - TableRowLevelMaintenancePlanningStatus::ManualReviewRequired - ); - assert_eq!(report.row_level_planning.delete_file_count, 2); - assert_eq!(report.row_level_planning.position_delete_file_count, 1); - assert_eq!(report.row_level_planning.equality_delete_file_count, 1); - assert!( - report - .row_level_planning - .reasons - .contains(&TableRowLevelMaintenancePlanningReason::DeleteFileRewriteUnsupported) - ); - assert!( - report - .row_level_planning - .delete_files - .iter() - .any(|delete_file| delete_file.file_location == position_delete_file - && delete_file.content == TableRowLevelDeleteFileContent::PositionDelete - && delete_file.object_exists) - ); - assert!( - report - .row_level_planning - .delete_files - .iter() - .any(|delete_file| delete_file.file_location == equality_delete_file - && delete_file.content == TableRowLevelDeleteFileContent::EqualityDelete - && delete_file.object_exists) - ); - let snapshot = compaction_snapshot_report(&report, 20); - assert_eq!(snapshot.status, TableCompactionPlanningStatus::ManualReviewRequired); - assert!(snapshot.reasons.contains(&TableCompactionPlanningReason::DeleteFile)); - assert!( - snapshot - .reasons - .contains(&TableCompactionPlanningReason::RowLevelRewriteUnsupported) - ); - - let err = store - .commit_table_compaction(bucket, "sales", "orders", config) - .await - .expect_err("delete-file compaction should fail closed"); - assert!( - err.to_string().contains("compaction has no safe rewrite candidates"), - "unexpected error: {err}" - ); - } - - #[tokio::test] - async fn compaction_plan_requires_current_snapshot_metadata() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 30, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 20, - "timestamp-ms": 2000, - "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" - } - ] - })) - .unwrap(), - ) - .await; - - let report = store - .plan_table_compaction( - bucket, - "sales", - "orders", - TableCompactionPlanningConfig { - target_file_size_bytes: 512 * 1024 * 1024, - small_file_threshold_bytes: 64 * 1024 * 1024, - min_input_files: 2, - max_rewrite_bytes_per_job: 1024 * 1024 * 1024, - }, - ) - .await - .expect("compaction planning should succeed"); - - assert_eq!(report.status, TableCompactionPlanningStatus::ManualReviewRequired); - assert_eq!(report.manual_review_count, 1); - let snapshot = compaction_snapshot_report(&report, 30); - assert!( - snapshot - .reasons - .contains(&TableCompactionPlanningReason::MissingCurrentSnapshot) - ); - } - - #[tokio::test] - async fn compaction_commit_rewrites_small_data_files_and_advances_pointer() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let data_dir = default_table_data_dir_path(&namespace, &table); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - let manifest_list = format!("{metadata_dir}/snap-20.avro"); - let manifest = format!("{metadata_dir}/manifest-20.avro"); - let left_data = format!("{data_dir}/part-left.parquet"); - let right_data = format!("{data_dir}/part-right.parquet"); - let retained_data = format!("{data_dir}/part-retained.parquet"); - let left_parquet = parquet_i32_bytes(&[1, 2]); - let right_parquet = parquet_i32_bytes(&[3, 4]); - let retained_values = (10..20_000).collect::>(); - let retained_parquet = parquet_i32_bytes(&retained_values); - let small_file_threshold_bytes = u64::try_from(left_parquet.len().max(right_parquet.len())).unwrap(); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0), (&retained_data, 0)]), - ) - .await; - backend.seed_object(bucket, &left_data, left_parquet).await; - backend.seed_object(bucket, &right_data, right_parquet).await; - backend.seed_object(bucket, &retained_data, retained_parquet).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://analytics/tables/table-id", - "last-sequence-number": 7, - "last-updated-ms": 2000, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 20, - "sequence-number": 7, - "timestamp-ms": 2000, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - ], - "current-snapshot-id": 20, - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .commit_table_compaction( - bucket, - "sales", - "orders", - TableCompactionPlanningConfig { - target_file_size_bytes: 64 * 1024, - small_file_threshold_bytes, - min_input_files: 2, - max_rewrite_bytes_per_job: 128 * 1024, - }, - ) - .await - .expect("compaction rewrite should commit"); - - assert_eq!(report.status, TableCompactionPlanningStatus::Committed); - assert_eq!(report.candidate_file_count, 2); - assert_eq!(report.rewrite_group_count, 1); - assert_eq!(report.manual_review_count, 0); - let committed_metadata = report - .committed_metadata_location - .as_ref() - .expect("compaction should report committed metadata"); - assert_ne!(committed_metadata, ¤t); - let rewrite_group = report.rewrite_groups.first().expect("rewrite group should be reported"); - assert_eq!(rewrite_group.input_file_locations, vec![left_data.clone(), right_data.clone()]); - let output_file = rewrite_group - .output_file_location - .as_ref() - .expect("rewrite group should include output data file"); - assert!(output_file.starts_with("s3://analytics/tables/table-id/data/")); - let output_file_key = - table_catalog_object_key_from_location(bucket, output_file).expect("output file should be inside the table bucket"); - assert!(output_file_key.starts_with("tables/table-id/data/")); - let output_object = backend - .read_object(bucket, &output_file_key) - .await - .unwrap() - .expect("compacted data file should be written"); - assert_eq!(parquet_i32_values(output_object.data), vec![1, 2, 3, 4]); - assert!(backend.object_exists(bucket, &left_data).await.unwrap()); - assert!(backend.object_exists(bucket, &right_data).await.unwrap()); - assert!(backend.object_exists(bucket, &retained_data).await.unwrap()); - - let table_entry = store - .load_table(bucket, "sales", "orders") - .await - .unwrap() - .expect("table should still exist"); - assert_eq!(table_entry.metadata_location, *committed_metadata); - assert_eq!(table_entry.generation, 2); - let metadata_object = backend - .read_object(bucket, committed_metadata) - .await - .unwrap() - .expect("compaction metadata should be written"); - let metadata = serde_json::from_slice::(&metadata_object.data).unwrap(); - assert_ne!(metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64), Some(20)); - assert_eq!(metadata.get("snapshots").and_then(serde_json::Value::as_array).unwrap().len(), 2); - assert_eq!( - metadata - .get("metadata-log") - .and_then(serde_json::Value::as_array) - .unwrap() - .last() - .and_then(|entry| entry.get("metadata-file")) - .and_then(serde_json::Value::as_str), - Some(current.as_str()) - ); - assert_eq!( - metadata - .get("snapshot-log") - .and_then(serde_json::Value::as_array) - .unwrap() - .last() - .and_then(|entry| entry.get("snapshot-id")) - .and_then(serde_json::Value::as_i64), - metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64) - ); - let current_manifest_list = metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .unwrap() - .last() - .and_then(|snapshot| snapshot.get("manifest-list")) - .and_then(serde_json::Value::as_str) - .unwrap(); - let manifest_list_object = backend - .read_object(bucket, current_manifest_list) - .await - .unwrap() - .expect("compaction manifest list should be written"); - let manifest_paths = manifest_paths_from_manifest_list_avro(&manifest_list_object.data).unwrap(); - assert_eq!(manifest_paths.len(), 1); - let manifest_object = backend - .read_object(bucket, &manifest_paths[0]) - .await - .unwrap() - .expect("compaction manifest should be written"); - let manifest_references = file_references_from_manifest_avro(&manifest_object.data).unwrap(); - let manifest_data_files = manifest_references - .into_iter() - .filter_map(|(location, kind)| (kind == TableMetadataMaintenanceObjectKind::DataFile).then_some(location)) - .collect::>(); - assert!(manifest_data_files.contains(output_file)); - assert!(manifest_data_files.contains(&retained_data)); - assert!(!manifest_data_files.contains(&left_data)); - assert!(!manifest_data_files.contains(&right_data)); - } - - #[tokio::test] - async fn compaction_commit_keeps_partition_rewrite_groups_isolated() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let data_dir = "tables/table-id/data"; - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - let manifest_list = format!("{metadata_dir}/snap-20.avro"); - let manifest = format!("{metadata_dir}/manifest-20.avro"); - let left_data = format!("{data_dir}/dt=2026-06-24/part-left.parquet"); - let right_data = format!("{data_dir}/dt=2026-06-24/part-right.parquet"); - let other_partition_data = format!("{data_dir}/dt=2026-06-25/part-only.parquet"); - let left_parquet = parquet_i32_bytes(&[1, 2]); - let right_parquet = parquet_i32_bytes(&[3, 4]); - let other_partition_parquet = parquet_i32_bytes(&[5, 6]); - let small_file_threshold_bytes = - u64::try_from(left_parquet.len().max(right_parquet.len()).max(other_partition_parquet.len())).unwrap(); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_dt_partition(&[ - (&left_data, 0, "2026-06-24"), - (&right_data, 0, "2026-06-24"), - (&other_partition_data, 0, "2026-06-25"), - ]), - ) - .await; - backend.seed_object(bucket, &left_data, left_parquet).await; - backend.seed_object(bucket, &right_data, right_parquet).await; - backend - .seed_object(bucket, &other_partition_data, other_partition_parquet) - .await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://analytics/tables/table-id", - "last-sequence-number": 7, - "last-updated-ms": 2000, - "schemas": [ - { - "schema-id": 0, - "type": "struct", - "fields": [ - {"id": 1, "name": "id", "required": true, "type": "int"}, - {"id": 2, "name": "dt", "required": false, "type": "string"} - ] - } - ], - "current-schema-id": 0, - "partition-specs": [ - { - "spec-id": 0, - "fields": [ - { - "source-id": 2, - "field-id": 1000, - "name": "dt", - "transform": "identity" - } - ] - } - ], - "default-spec-id": 0, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 20, - "sequence-number": 7, - "timestamp-ms": 2000, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - ], - "current-snapshot-id": 20, - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .commit_table_compaction( - bucket, - "sales", - "orders", - TableCompactionPlanningConfig { - target_file_size_bytes: 64 * 1024, - small_file_threshold_bytes, - min_input_files: 2, - max_rewrite_bytes_per_job: 128 * 1024, - }, - ) - .await - .expect("partition-local compaction rewrite should commit"); - - assert_eq!(report.status, TableCompactionPlanningStatus::Committed); - assert_eq!(report.candidate_file_count, 3); - assert_eq!(report.rewrite_group_count, 1); - let rewrite_group = report.rewrite_groups.first().expect("rewrite group should be reported"); - assert_eq!(rewrite_group.input_file_locations, vec![left_data.clone(), right_data.clone()]); - let output_file = rewrite_group - .output_file_location - .as_ref() - .expect("rewrite group should include output data file"); - assert!(output_file.starts_with("s3://analytics/tables/table-id/data/dt=2026-06-24/")); - let output_file_key = - table_catalog_object_key_from_location(bucket, output_file).expect("output file should be inside the table bucket"); - let output_object = backend - .read_object(bucket, &output_file_key) - .await - .unwrap() - .expect("compacted partition data file should be written"); - assert_eq!(parquet_i32_values(output_object.data), vec![1, 2, 3, 4]); - assert!(backend.object_exists(bucket, &other_partition_data).await.unwrap()); - - let table_entry = store - .load_table(bucket, "sales", "orders") - .await - .unwrap() - .expect("table should still exist"); - let metadata_object = backend - .read_object(bucket, &table_entry.metadata_location) - .await - .unwrap() - .expect("compaction metadata should be written"); - let metadata = serde_json::from_slice::(&metadata_object.data).unwrap(); - let current_manifest_list = metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .unwrap() - .last() - .and_then(|snapshot| snapshot.get("manifest-list")) - .and_then(serde_json::Value::as_str) - .unwrap(); - let manifest_list_object = backend - .read_object(bucket, current_manifest_list) - .await - .unwrap() - .expect("compaction manifest list should be written"); - let manifest_references = manifest_list_references_from_manifest_list_avro(&manifest_list_object.data).unwrap(); - assert_eq!(manifest_references.len(), 1); - assert_eq!(manifest_references[0].partition_spec_id, Some(0)); - let manifest_object = backend - .read_object(bucket, &manifest_references[0].manifest_path) - .await - .unwrap() - .expect("compaction manifest should be written"); - let data_file_references = data_file_references_from_manifest_avro(&manifest_object.data).unwrap(); - let output_reference = data_file_references - .iter() - .find(|reference| reference.location == *output_file) - .expect("compacted output should be present in the manifest"); - assert_eq!( - output_reference.partition, - vec![("dt".to_string(), apache_avro::types::Value::String("2026-06-24".to_string()))] - ); - let retained_reference = data_file_references - .iter() - .find(|reference| reference.location == other_partition_data) - .expect("retained partition file should stay in the manifest"); - assert_eq!( - retained_reference.partition, - vec![("dt".to_string(), apache_avro::types::Value::String("2026-06-25".to_string()))] - ); - } - - #[tokio::test] - async fn compaction_commit_preserves_sort_order_and_keeps_groups_isolated() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let data_dir = default_table_data_dir_path(&namespace, &table); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - let manifest_list = format!("{metadata_dir}/snap-20.avro"); - let manifest = format!("{metadata_dir}/manifest-20.avro"); - let left_data = format!("{data_dir}/part-left.parquet"); - let right_data = format!("{data_dir}/part-right.parquet"); - let other_sort_data = format!("{data_dir}/part-other-sort.parquet"); - let left_parquet = parquet_i32_bytes(&[1, 2]); - let right_parquet = parquet_i32_bytes(&[3, 4]); - let other_sort_parquet = parquet_i32_bytes(&[5, 6]); - let small_file_threshold_bytes = - u64::try_from(left_parquet.len().max(right_parquet.len()).max(other_sort_parquet.len())).unwrap(); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_sort_order(&[(&left_data, 0, 7), (&right_data, 0, 7), (&other_sort_data, 0, 8)]), - ) - .await; - backend.seed_object(bucket, &left_data, left_parquet).await; - backend.seed_object(bucket, &right_data, right_parquet).await; - backend.seed_object(bucket, &other_sort_data, other_sort_parquet).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://analytics/tables/table-id", - "last-sequence-number": 7, - "last-updated-ms": 2000, - "sort-orders": [ - {"order-id": 7, "fields": []}, - {"order-id": 8, "fields": []} - ], - "default-sort-order-id": 7, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 20, - "sequence-number": 7, - "timestamp-ms": 2000, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - ], - "current-snapshot-id": 20, - "refs": { - "main": { - "snapshot-id": 20, - "type": "branch" - } - } - })) - .unwrap(), - ) - .await; - - let report = store - .commit_table_compaction( - bucket, - "sales", - "orders", - TableCompactionPlanningConfig { - target_file_size_bytes: 64 * 1024, - small_file_threshold_bytes, - min_input_files: 2, - max_rewrite_bytes_per_job: 128 * 1024, - }, - ) - .await - .expect("sort-aware compaction rewrite should commit"); - - assert_eq!(report.status, TableCompactionPlanningStatus::Committed); - assert_eq!(report.candidate_file_count, 3); - assert_eq!(report.rewrite_group_count, 1); - let rewrite_group = report.rewrite_groups.first().expect("rewrite group should be reported"); - assert_eq!(rewrite_group.sort_order_id, Some(7)); - assert_eq!(rewrite_group.input_file_locations, vec![left_data.clone(), right_data.clone()]); - let output_file = rewrite_group - .output_file_location - .as_ref() - .expect("rewrite group should include output data file"); - - let table_entry = store - .load_table(bucket, "sales", "orders") - .await - .unwrap() - .expect("table should still exist"); - let metadata_object = backend - .read_object(bucket, &table_entry.metadata_location) - .await - .unwrap() - .expect("compaction metadata should be written"); - let metadata = serde_json::from_slice::(&metadata_object.data).unwrap(); - let current_manifest_list = metadata - .get("snapshots") - .and_then(serde_json::Value::as_array) - .unwrap() - .last() - .and_then(|snapshot| snapshot.get("manifest-list")) - .and_then(serde_json::Value::as_str) - .unwrap(); - let manifest_list_object = backend - .read_object(bucket, current_manifest_list) - .await - .unwrap() - .expect("compaction manifest list should be written"); - let manifest_references = manifest_list_references_from_manifest_list_avro(&manifest_list_object.data).unwrap(); - let manifest_object = backend - .read_object(bucket, &manifest_references[0].manifest_path) - .await - .unwrap() - .expect("compaction manifest should be written"); - let data_file_references = data_file_references_from_manifest_avro(&manifest_object.data).unwrap(); - let output_reference = data_file_references - .iter() - .find(|reference| reference.location == *output_file) - .expect("compacted output should be present in the manifest"); - assert_eq!(output_reference.sort_order_id, Some(7)); - let retained_reference = data_file_references - .iter() - .find(|reference| reference.location == other_sort_data) - .expect("different sort order file should stay in the manifest"); - assert_eq!(retained_reference.sort_order_id, Some(8)); - } - - #[tokio::test] - async fn compaction_commit_rejects_schema_mismatch_without_advancing_pointer() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let data_dir = default_table_data_dir_path(&namespace, &table); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - let manifest_list = format!("{metadata_dir}/snap-20.avro"); - let manifest = format!("{metadata_dir}/manifest-20.avro"); - let left_data = format!("{data_dir}/part-left.parquet"); - let right_data = format!("{data_dir}/part-right.parquet"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0)])) - .await; - backend.seed_object(bucket, &left_data, parquet_i32_bytes(&[1, 2])).await; - backend.seed_object(bucket, &right_data, parquet_i64_bytes(&[3, 4])).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://analytics/tables/table-id", - "last-sequence-number": 7, - "last-updated-ms": 2000, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 20, - "sequence-number": 7, - "timestamp-ms": 2000, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - ], - "current-snapshot-id": 20 - })) - .unwrap(), - ) - .await; - - let error = store - .commit_table_compaction( - bucket, - "sales", - "orders", - TableCompactionPlanningConfig { - target_file_size_bytes: 64 * 1024, - small_file_threshold_bytes: 32 * 1024, - min_input_files: 2, - max_rewrite_bytes_per_job: 128 * 1024, - }, - ) - .await - .expect_err("schema mismatch should reject compaction commit"); - - assert!(matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("schemas must match"))); - let table_entry = store - .load_table(bucket, "sales", "orders") - .await - .unwrap() - .expect("table should still exist"); - assert_eq!(table_entry.metadata_location, current); - } - - #[tokio::test] - async fn compaction_commit_rejects_deleted_manifest_entries_without_advancing_pointer() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let metadata_dir = default_table_metadata_dir_path(&namespace, &table); - let data_dir = default_table_data_dir_path(&namespace, &table); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - let manifest_list = format!("{metadata_dir}/snap-20.avro"); - let manifest = format!("{metadata_dir}/manifest-20.avro"); - let left_data = format!("{data_dir}/part-left.parquet"); - let deleted_data = format!("{data_dir}/part-deleted.parquet"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_status(&[(&left_data, 0, 1), (&deleted_data, 0, 2)]), - ) - .await; - backend.seed_object(bucket, &left_data, parquet_i32_bytes(&[1, 2])).await; - backend.seed_object(bucket, &deleted_data, parquet_i32_bytes(&[3, 4])).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "format-version": 2, - "table-uuid": "table-uuid", - "location": "s3://analytics/tables/table-id", - "last-sequence-number": 7, - "last-updated-ms": 2000, - "metadata-log": [], - "snapshots": [ - { - "snapshot-id": 20, - "sequence-number": 7, - "timestamp-ms": 2000, - "manifest-list": manifest_list, - "summary": { - "operation": "overwrite" - } - } - ], - "current-snapshot-id": 20 - })) - .unwrap(), - ) - .await; - - let error = store - .commit_table_compaction( - bucket, - "sales", - "orders", - TableCompactionPlanningConfig { - target_file_size_bytes: 64 * 1024, - small_file_threshold_bytes: 32 * 1024, - min_input_files: 2, - max_rewrite_bytes_per_job: 128 * 1024, - }, - ) - .await - .expect_err("deleted manifest entries should reject compaction commit"); - - assert!(matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("no safe rewrite candidates"))); - let table_entry = store - .load_table(bucket, "sales", "orders") - .await - .unwrap() - .expect("table should still exist"); - assert_eq!(table_entry.metadata_location, current); - } - - #[tokio::test] - async fn maintenance_dry_run_keeps_recent_metadata_files_and_ignores_non_metadata_objects() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let v1 = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let v2 = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let recent = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); - let manifest = format!("{}/snap-1.avro", default_table_metadata_dir_path(&namespace, &table)); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - for metadata in [&v1, &v2, &recent] { - backend.seed_object(bucket, metadata, b"{}".to_vec()).await; - } - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - backend.seed_object(bucket, &manifest, b"manifest".to_vec()).await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 2) - .await - .unwrap(); - - assert!(report.retained_metadata_locations.contains(&recent)); - assert!(report.retained_metadata_locations.contains(¤t)); - assert_eq!(report.cleanup_candidate_locations, vec![v1, v2]); - assert!(!report.cleanup_candidate_locations.contains(&manifest)); - } - - #[tokio::test] - async fn maintenance_delete_removes_only_dry_run_metadata_candidates() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let retained = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - let manifest = format!("{}/snap-1.avro", default_table_metadata_dir_path(&namespace, &table)); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend.seed_object(bucket, &retained, b"{}".to_vec()).await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "metadata-log": [ - { - "timestamp-ms": 1, - "metadata-file": retained - } - ], - "snapshots": [ - { - "snapshot-id": 1, - "manifest-list": manifest - } - ] - })) - .unwrap(), - ) - .await; - backend.seed_object(bucket, &manifest, manifest_list_avro_bytes(&[])).await; - - let report = store - .delete_table_metadata_maintenance_candidates(bucket, "sales", "orders", 0) - .await - .unwrap(); - - assert_eq!(report.cleanup_candidate_locations, vec![old.clone()]); - assert!(report.cleanup_object_candidate_locations.is_empty()); - assert!(!backend.object_exists(bucket, &old).await.unwrap()); - assert!(backend.object_exists(bucket, &retained).await.unwrap()); - assert!(backend.object_exists(bucket, ¤t).await.unwrap()); - assert!(backend.object_exists(bucket, &manifest).await.unwrap()); - } - - #[tokio::test] - async fn maintenance_delete_skips_recent_uncommitted_metadata_candidates() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let fresh = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - backend - .seed_object_with_mod_time(bucket, &fresh, br#"{"metadata-log":[]}"#.to_vec(), Some(OffsetDateTime::now_utc())) - .await; - - let report = store - .delete_table_metadata_maintenance_candidates(bucket, "sales", "orders", 0) - .await - .unwrap(); - - assert_eq!(report.cleanup_candidate_locations, vec![old.clone()]); - assert!(!backend.object_exists(bucket, &old).await.unwrap()); - assert!(backend.object_exists(bucket, &fresh).await.unwrap()); - } - - #[tokio::test] - async fn maintenance_delete_does_not_expand_beyond_planned_deletable_candidates() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let fresh = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - backend - .seed_object_with_mod_time(bucket, &fresh, b"{}".to_vec(), Some(OffsetDateTime::now_utc())) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .unwrap(); - assert_eq!(report.cleanup_candidate_locations, vec![old.clone(), fresh.clone()]); - assert_eq!(report.deletable_metadata_locations, vec![old.clone()]); - - backend.seed_object(bucket, &fresh, b"{}".to_vec()).await; - let deleted = store - .delete_table_metadata_maintenance_report(bucket, "sales", "orders", report) - .await - .unwrap(); - - assert_eq!(deleted.job.operation, TableMetadataMaintenanceOperation::Delete); - assert_eq!(deleted.job.status, TableMetadataMaintenanceJobStatus::Successful); - assert_eq!(deleted.job.deleted_metadata_file_count, 1); - assert_eq!(deleted.cleanup_candidate_locations, vec![old.clone()]); - assert_eq!(deleted.deletable_metadata_locations, vec![old.clone()]); - assert_eq!( - maintenance_object_report(&deleted, &old).state, - TableMetadataMaintenanceObjectState::Deleted - ); - assert_eq!( - maintenance_object_report(&deleted, &fresh).state, - TableMetadataMaintenanceObjectState::PendingSafetyWindow - ); - assert!(!backend.object_exists(bucket, &old).await.unwrap()); - assert!(backend.object_exists(bucket, &fresh).await.unwrap()); - } - - #[tokio::test] - async fn maintenance_delete_conflicts_when_current_pointer_changes_before_delete() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let next = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &old, b"{}".to_vec()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - - let report = store - .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) - .await - .unwrap(); - assert_eq!(report.cleanup_candidate_locations, vec![old.clone()]); - - backend.seed_object(bucket, &next, br#"{"metadata-log":[]}"#.to_vec()).await; - store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-id".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current, - new_metadata_location: next, - requirements: Vec::new(), - writer: Some("test".to_string()), - }) - .await - .unwrap(); - - let err = store - .delete_table_metadata_maintenance_report(bucket, "sales", "orders", report) - .await - .unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Conflict(_)); - assert!(backend.object_exists(bucket, &old).await.unwrap()); - } - - #[tokio::test] - async fn export_catalog_entry_contains_table_identity_and_pointer() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - - let export = store.export_table_catalog_entry(bucket, "sales", "orders").await.unwrap(); - - assert_eq!(export.table_bucket.table_bucket, bucket); - assert_eq!(export.namespace.namespace, "sales"); - assert_eq!(export.table.namespace, "sales"); - assert_eq!(export.table.table, "orders"); - assert_eq!(export.table.table_id, "table-id"); - assert_eq!(export.table.table_uuid, "table-uuid"); - assert_eq!(export.table.metadata_location, current); - assert_eq!(export.table.version_token, "token-v1"); - assert_eq!(export.table.generation, 1); - } - - #[tokio::test] - async fn export_catalog_entry_includes_backing_migration_manifest() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - - let export = store.export_table_catalog_entry(bucket, "sales", "orders").await.unwrap(); - - assert_eq!(export.backing_manifest.version, TABLE_CATALOG_BACKING_MANIFEST_VERSION); - assert_eq!(export.backing_manifest.current.kind, TableCatalogBackingKind::ObjectBacked); - assert_eq!(export.backing_manifest.current.authority, TableCatalogAuthority::RustfsSysObject); - assert_eq!( - export.backing_manifest.current.consistency, - TableCatalogConsistencyMode::ConditionalObjectCas - ); - assert_eq!(export.backing_manifest.current.wal.finalization_required_count, 0); - assert_eq!(export.backing_manifest.migration.target_kind, TableCatalogBackingKind::StrongKvWal); - assert_eq!( - export.backing_manifest.migration.status, - TableCatalogBackingMigrationStatus::ReadyToSnapshot - ); - assert!( - export - .backing_manifest - .migration - .required_steps - .contains(&TableCatalogBackingMigrationStep::ReplayCommitLog) - ); - assert_eq!( - export.backing_manifest.ha.writer_region_model, - TableCatalogHaWriterModel::SingleActiveWriterRegion - ); - assert!(!export.backing_manifest.ha.active_active_supported); - } - - #[tokio::test] - async fn object_table_catalog_store_serializes_namespace_drop_with_table_creation() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - - let table_path = store.paths.table_entry_path(bucket, &namespace, &table); - let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &table_path).await; - let create_store = store.clone(); - let table_entry = test_table_entry(bucket, &namespace, &table, metadata_location); - let create = tokio::spawn(async move { create_store.create_table(table_entry).await }); - pause.wait_started().await; - - let drop_store = store.clone(); - let namespace_name = namespace.public_name(); - let mut drop_namespace = tokio::spawn(async move { drop_store.drop_namespace(bucket, &namespace_name).await }); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(50), &mut drop_namespace) - .await - .is_err() - ); - - pause.release(); - create.await.unwrap().unwrap(); - assert_matches!( - drop_namespace.await.unwrap(), - Err(TableCatalogStoreError::Conflict(message)) if message.contains("is not empty") - ); - assert!( - store - .load_table(bucket, &namespace.public_name(), table.as_str()) - .await - .unwrap() - .is_some() - ); - } - - #[tokio::test] - async fn durable_strong_migration_dry_run_reports_ready_catalog_inventory() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let view = IdentifierSegment::parse("recent_orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let view_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata.clone()).await; - store - .create_view(test_view_entry(bucket, &namespace, &view, view_metadata)) - .await - .unwrap(); - backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; - store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: next_metadata, - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap(); - - let report = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); - - assert_eq!(report.table_bucket, bucket); - assert_eq!(report.source_kind, TableCatalogBackingKind::ObjectBacked); - assert_eq!(report.target_kind, TableCatalogBackingKind::StrongKvWal); - assert_eq!(report.status, TableCatalogBackingMigrationStatus::ReadyToSnapshot); - assert_eq!(report.namespace_count, 1); - assert_eq!(report.table_count, 1); - assert_eq!(report.view_count, 1); - assert_eq!(report.commit_log_count, 1); - assert_eq!(report.idempotency_index_count, 1); - assert_eq!(report.warehouse_prefix_count, 1); - assert!(!report.object_backed_writes_fenced); - assert!(!report.ready_to_enable_durable_strong); - assert!(report.blockers.is_empty()); - assert!( - report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::SnapshotObjectBackedCatalog) - ); - assert!( - !report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::EnableDurableStrongBacking) - ); - assert_eq!(report.rollback.backing_config_key, ENV_TABLE_CATALOG_BACKING); - assert_eq!(report.rollback.rollback_backing_value, TABLE_CATALOG_BACKING_OBJECT); - } - - #[tokio::test] - async fn durable_strong_migration_rejects_orphan_table_without_namespace() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; - - let namespace_path = object_store.paths.namespace_entry_path(bucket, &namespace); - backend - .delete_object(RUSTFS_META_BUCKET, &namespace_path) - .await - .expect("namespace marker should be removed"); - assert!( - object_store - .load_table(bucket, &namespace.public_name(), table.as_str()) - .await - .unwrap() - .is_some() - ); - - let error = object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap_err(); - assert_matches!( - error, - TableCatalogStoreError::Invalid(message) if message.contains("table entry has no namespace entry") - ); - object_store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("failed migration must leave source catalog writable"); - object_store.put_table_bucket(test_bucket_entry("research")).await.unwrap(); - } - - #[tokio::test] - async fn durable_strong_migration_rejects_orphan_view_without_namespace() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); - let metadata_location = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); - - object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - object_store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - object_store - .create_view(test_view_entry(bucket, &namespace, &view, metadata_location)) - .await - .unwrap(); - object_store.backfill_table_warehouse_index(bucket).await.unwrap(); - - let namespace_path = object_store.paths.namespace_entry_path(bucket, &namespace); - backend - .delete_object(RUSTFS_META_BUCKET, &namespace_path) - .await - .expect("namespace marker should be removed"); - assert!( - object_store - .load_view(bucket, &namespace.public_name(), view.as_str()) - .await - .unwrap() - .is_some() - ); - - let error = object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap_err(); - assert_matches!( - error, - TableCatalogStoreError::Invalid(message) if message.contains("view entry has no namespace entry") - ); - object_store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("failed migration must leave source catalog writable"); - } - - #[tokio::test] - async fn durable_strong_migration_rejects_orphan_idempotency_without_fencing_source() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata.clone()).await; - - let orphan = CommitLogEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - commit_id: "orphan-commit".to_string(), - idempotency_key: Some("orphan-request".to_string()), - table_id: "table-id".to_string(), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - new_version_token: "token-v2".to_string(), - previous_metadata_location: current_metadata.clone(), - new_metadata_location: current_metadata, - requirements: Vec::new(), - status: CommitLogStatus::Committed, - writer: None, - created_at: None, - updated_at: None, - }; - let idempotency_path = object_store - .paths - .commit_idempotency_entry_path(bucket, "table-id", "orphan-request"); - backend - .seed_object(RUSTFS_META_BUCKET, &idempotency_path, serde_json::to_vec(&orphan).unwrap()) - .await; - - let error = object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap_err(); - assert_matches!( - error, - TableCatalogStoreError::Invalid(message) if message.contains("has no commit record") - ); - - object_store - .create_namespace(test_namespace_entry(bucket, &Namespace::parse("still_writable").unwrap())) - .await - .unwrap(); - object_store.put_table_bucket(test_bucket_entry("research")).await.unwrap(); - } - - #[tokio::test] - async fn durable_strong_migration_materializes_catalog_state_and_fences_object_writes() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let view = IdentifierSegment::parse("recent_orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let strong_metadata = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - let view_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata.clone()).await; - object_store - .create_view(test_view_entry(bucket, &namespace, &view, view_metadata)) - .await - .unwrap(); - backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; - let object_commit = object_store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "object-commit".to_string(), - idempotency_key: Some("object-request".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: next_metadata.clone(), - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap(); - - let materialized = object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap(); - assert_eq!(materialized.status, TableCatalogBackingMigrationExecutionStatus::SnapshotMaterialized); - assert_eq!(materialized.namespace_count, 1); - assert_eq!(materialized.table_count, 1); - assert_eq!(materialized.view_count, 1); - assert_eq!(materialized.commit_log_count, 1); - assert_eq!(materialized.idempotency_index_count, 1); - assert!(materialized.object_backed_writes_fenced); - assert!(materialized.ready_to_enable_durable_strong); - - let retry = object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap(); - assert_eq!(retry.status, TableCatalogBackingMigrationExecutionStatus::SnapshotAlreadyMaterialized); - assert_eq!(retry.source_fingerprint, materialized.source_fingerprint); - - let migration = object_store.plan_durable_strong_backing_migration(bucket).await.unwrap(); - assert_eq!(migration.status, TableCatalogBackingMigrationStatus::SnapshotMaterialized); - assert!(migration.object_backed_writes_fenced); - assert!(migration.ready_to_enable_durable_strong); - assert!( - migration - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::EnableDurableStrongBacking) - ); - - let strong_store = StrongTableCatalogStore::new(backend.clone()); - let migrated_table = strong_store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(migrated_table.metadata_location, next_metadata); - assert_eq!(migrated_table.version_token, object_commit.table.version_token); - assert_eq!(migrated_table.generation, 2); - assert!( - strong_store - .load_view(bucket, "sales", "recent_orders") - .await - .unwrap() - .is_some() - ); - assert!( - strong_store - .get_commit_by_id(bucket, "table-id", "object-commit") - .await - .unwrap() - .is_some() - ); - assert!( - strong_store - .get_commit_by_idempotency_key(bucket, "table-id", "object-request") - .await - .unwrap() - .is_some() - ); - - let object_err = object_store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "stale-object-commit".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: migrated_table.version_token.clone(), - expected_metadata_location: migrated_table.metadata_location.clone(), - new_metadata_location: strong_metadata.clone(), - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap_err(); - assert_matches!(object_err, TableCatalogStoreError::Conflict(message) if message.contains("writes are fenced")); - - backend.seed_object(bucket, &strong_metadata, b"{}".to_vec()).await; - let strong_commit = strong_store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "strong-commit".to_string(), - idempotency_key: Some("strong-request".to_string()), - operation: "append".to_string(), - expected_version_token: migrated_table.version_token, - expected_metadata_location: migrated_table.metadata_location, - new_metadata_location: strong_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap(); - assert_eq!(strong_commit.table.generation, 3); - - let cancel_err = object_store - .cancel_durable_strong_backing_migration(bucket) - .await - .unwrap_err(); - assert_matches!(cancel_err, TableCatalogStoreError::Conflict(message) if message.contains("changed after materializing")); - } - - #[tokio::test] - async fn durable_strong_migration_cancel_restores_object_backed_writes_before_target_changes() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata.clone()).await; - object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap(); - - let cancelled = object_store.cancel_durable_strong_backing_migration(bucket).await.unwrap(); - assert_eq!(cancelled.status, TableCatalogBackingMigrationCancelStatus::FenceReleased); - assert!(!cancelled.object_backed_writes_fenced); - assert!( - StrongTableCatalogStore::new(backend.clone()) - .load_table(bucket, "sales", "orders") - .await - .unwrap() - .is_none() - ); - object_store.put_table_bucket(test_bucket_entry("research")).await.unwrap(); - - backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; - let result = object_store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "rollback-object-commit".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: next_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap(); - assert_eq!(result.table.generation, 2); - } - - #[tokio::test] - async fn durable_strong_migration_waits_for_in_flight_catalog_writes() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - object_store.backfill_table_warehouse_index(bucket).await.unwrap(); - - let namespace_path = object_store.paths.namespace_entry_path(bucket, &namespace); - let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &namespace_path).await; - let namespace_store = object_store.clone(); - let namespace_write = tokio::spawn(async move { - namespace_store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - }); - pause.wait_started().await; - - let migration_store = object_store.clone(); - let mut migration = - tokio::spawn(async move { migration_store.materialize_durable_strong_backing_migration(bucket).await }); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(50), &mut migration) - .await - .is_err() - ); - - pause.release(); - namespace_write.await.unwrap().unwrap(); - let report = migration.await.unwrap().unwrap(); - assert_eq!(report.namespace_count, 1); - assert!( - StrongTableCatalogStore::new(backend) - .get_namespace(bucket, "sales") - .await - .unwrap() - .is_some() - ); - } - - #[tokio::test] - async fn durable_strong_migration_retry_recovers_after_fence_finalization_failure() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; - - let fence_path = object_store.paths.backing_migration_fence_path(bucket); - backend.fail_put_attempt(RUSTFS_META_BUCKET, &fence_path, 2).await; - let first = object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap_err(); - assert_matches!(first, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); - assert_matches!( - object_store - .create_namespace(test_namespace_entry(bucket, &Namespace::parse("blocked").unwrap())) - .await - .unwrap_err(), - TableCatalogStoreError::Conflict(message) if message.contains("writes are fenced") - ); - - let retry = object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap(); - assert_eq!(retry.status, TableCatalogBackingMigrationExecutionStatus::SnapshotAlreadyMaterialized); - assert!(retry.ready_to_enable_durable_strong); - } - - #[tokio::test] - async fn durable_strong_migration_cancel_preserves_preexisting_target_state() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; - - let strong_store = StrongTableCatalogStore::new(backend.clone()); - strong_store - .put_table_bucket(object_store.get_table_bucket(bucket).await.unwrap().unwrap()) - .await - .unwrap(); - strong_store - .create_namespace(object_store.get_namespace(bucket, "sales").await.unwrap().unwrap()) - .await - .unwrap(); - strong_store - .create_table(object_store.load_table(bucket, "sales", "orders").await.unwrap().unwrap()) - .await - .unwrap(); - - let materialized = object_store - .materialize_durable_strong_backing_migration(bucket) - .await - .unwrap(); - assert_eq!( - materialized.status, - TableCatalogBackingMigrationExecutionStatus::SnapshotAlreadyMaterialized - ); - object_store.cancel_durable_strong_backing_migration(bucket).await.unwrap(); - assert!(strong_store.load_table(bucket, "sales", "orders").await.unwrap().is_some()); - } - - #[tokio::test] - async fn durable_strong_migration_requires_every_table_bucket_before_cutover() { - let backend = TestCatalogObjectBackend::default(); - let object_store = ObjectTableCatalogStore::new(backend); - for bucket in ["analytics", "research"] { - object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - object_store.backfill_table_warehouse_index(bucket).await.unwrap(); - } - - let first = object_store - .materialize_durable_strong_backing_migration("analytics") - .await - .unwrap(); - assert!(!first.ready_to_enable_durable_strong); - assert_matches!( - object_store - .put_table_bucket(test_bucket_entry("new-bucket")) - .await - .unwrap_err(), - TableCatalogStoreError::Conflict(message) if message.contains("registry writes are fenced") - ); - assert!( - object_store - .plan_durable_strong_backing_migration("analytics") - .await - .unwrap() - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::SnapshotRemainingTableBuckets) - ); - - let second = object_store - .materialize_durable_strong_backing_migration("research") - .await - .unwrap(); - assert!(second.ready_to_enable_durable_strong); - assert!( - object_store - .plan_durable_strong_backing_migration("analytics") - .await - .unwrap() - .ready_to_enable_durable_strong - ); - } - - #[tokio::test] - async fn durable_strong_migration_dry_run_reports_recovery_blockers() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata.clone()).await; - backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; - store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: next_metadata, - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap(); - let idempotency_path = store - .paths - .commit_idempotency_entry_path(bucket, "table-id", "client-request"); - backend.delete_object(RUSTFS_META_BUCKET, &idempotency_path).await.unwrap(); - - let report = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); - - assert_eq!(report.status, TableCatalogBackingMigrationStatus::RecoveryRequired); - assert_eq!(report.commit_log_count, 1); - assert_eq!(report.idempotency_index_count, 0); - assert!( - report - .blockers - .contains(&TableCatalogBackingMigrationBlocker::CommitRecoveryRequired) - ); - assert!( - report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::RunCatalogRecovery) - ); - assert!( - !report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::SnapshotObjectBackedCatalog) - ); - assert!( - !report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::EnableDurableStrongBacking) - ); - assert!( - !report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::VerifyDurableStrongSnapshot) - ); - } - - #[tokio::test] - async fn durable_strong_migration_dry_run_requires_ready_warehouse_index() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata).await; - let state_path = store.paths.warehouse_index_state_path(bucket); - backend.delete_object(RUSTFS_META_BUCKET, &state_path).await.unwrap(); - - let report = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); - - assert_eq!(report.status, TableCatalogBackingMigrationStatus::RecoveryRequired); - assert!(!report.warehouse_index_ready); - assert!( - report - .blockers - .contains(&TableCatalogBackingMigrationBlocker::WarehouseIndexBackfillRequired) - ); - assert!( - report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::BackfillWarehouseIndex) - ); - assert!( - !report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::SnapshotObjectBackedCatalog) - ); - assert!( - !report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::EnableDurableStrongBacking) - ); - assert!( - !report - .recommended_actions - .contains(&TableCatalogBackingMigrationAction::VerifyDurableStrongSnapshot) - ); - } - - #[tokio::test] - async fn strong_catalog_backing_commit_is_atomic_with_wal_and_idempotency() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - - let result = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata.clone(), - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap(); - - assert_eq!(result.table.metadata_location, new_metadata); - assert_eq!(result.table.generation, 2); - assert_eq!(result.commit_log.status, CommitLogStatus::Committed); - - let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(loaded.metadata_location, result.table.metadata_location); - assert_eq!(loaded.version_token, result.table.version_token); - assert_eq!( - store - .get_commit_by_id(bucket, "table-id", "commit-1") - .await - .unwrap() - .unwrap() - .status, - CommitLogStatus::Committed - ); - assert_eq!( - store - .get_commit_by_idempotency_key(bucket, "table-id", "client-request") - .await - .unwrap() - .unwrap() - .status, - CommitLogStatus::Committed - ); - - let recovery = store.plan_table_commit_recovery(bucket, "sales", "orders").await.unwrap(); - assert_eq!(recovery.staged_before_table_update_count, 0); - assert_eq!(recovery.finalization_required_count, 0); - assert_eq!(recovery.idempotency_repair_required_count, 0); - assert_eq!(recovery.manual_review_count, 0); - } - - #[tokio::test] - async fn strong_catalog_pagination_uses_ordered_state_and_rejects_object_cursors() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let namespace_name = namespace.public_name(); - let one = NonZeroUsize::new(1).expect("page size should be non-zero"); - - seed_catalog_list_entries(&store, bucket, &namespace).await; - - let first = store - .list_namespaces_page(bucket, None, one) - .await - .expect("first strong namespace page should load"); - assert_eq!(first.entries[0].namespace, "analytics"); - let second = store - .list_namespaces_page(bucket, first.next_cursor.as_deref(), one) - .await - .expect("second strong namespace page should load"); - assert_eq!(second.entries[0].namespace, "sales"); - assert!(second.next_cursor.is_none()); - - let first = store - .list_tables_page(bucket, &namespace_name, None, one) - .await - .expect("first strong table page should load"); - assert_eq!(first.entries[0].table, "alpha"); - assert!(first.next_cursor.as_deref().is_some_and(|cursor| cursor == "strong:alpha")); - let second = store - .list_tables_page(bucket, &namespace_name, first.next_cursor.as_deref(), one) - .await - .expect("second strong table page should load"); - assert_eq!(second.entries[0].table, "beta"); - assert!(second.next_cursor.is_none()); - - let first = store - .list_views_page(bucket, &namespace_name, None, one) - .await - .expect("first strong view page should load"); - assert_eq!(first.entries[0].view, "alpha"); - let second = store - .list_views_page(bucket, &namespace_name, first.next_cursor.as_deref(), one) - .await - .expect("second strong view page should load"); - assert_eq!(second.entries[0].view, "beta"); - assert!(second.next_cursor.is_none()); - - let exact = NonZeroUsize::new(2).expect("exact page size should be non-zero"); - assert!( - store - .list_namespaces_page(bucket, None, exact) - .await - .expect("exact strong namespace page should load") - .next_cursor - .is_none() - ); - assert!( - store - .list_tables_page(bucket, &namespace_name, None, exact) - .await - .expect("exact strong table page should load") - .next_cursor - .is_none() - ); - assert!( - store - .list_views_page(bucket, &namespace_name, None, exact) - .await - .expect("exact strong view page should load") - .next_cursor - .is_none() - ); - - assert!(matches!( - store - .list_tables_page(bucket, &namespace_name, Some("object:alpha"), one) - .await, - Err(TableCatalogStoreError::Invalid(_)) - )); - } - - #[tokio::test] - async fn strong_catalog_backing_replays_durable_commit_state_after_restart() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("namespace should be created"); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .expect("table should be created"); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - - let result = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata.clone(), - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .expect("commit should succeed"); - - let restarted = StrongTableCatalogStore::new(backend.clone()); - let loaded = restarted - .load_table(bucket, "sales", "orders") - .await - .expect("table load should succeed") - .expect("table should replay from durable state"); - assert_eq!(loaded.metadata_location, result.table.metadata_location); - assert_eq!(loaded.version_token, result.table.version_token); - assert_eq!( - restarted - .get_commit_by_id(bucket, "table-id", "commit-1") - .await - .expect("commit lookup should succeed") - .expect("commit log should replay from durable state") - .status, - CommitLogStatus::Committed - ); - assert_eq!( - restarted - .get_commit_by_idempotency_key(bucket, "table-id", "client-request") - .await - .expect("idempotency lookup should succeed") - .expect("idempotency index should replay from durable state") - .status, - CommitLogStatus::Committed - ); - - let recovery = restarted - .plan_table_commit_recovery(bucket, "sales", "orders") - .await - .expect("recovery report should replay from durable state"); - assert_eq!(recovery.finalized_count, 1); - assert_eq!(recovery.finalization_required_count, 0); - assert_eq!(recovery.idempotency_repair_required_count, 0); - assert_eq!(recovery.manual_review_count, 0); - } - - #[tokio::test] - async fn strong_catalog_backing_rejects_stale_snapshot_cas_after_concurrent_restart() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let first_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let stale_metadata = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("namespace should be created"); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .expect("table should be created"); - backend.seed_object(bucket, &first_metadata, b"{}".to_vec()).await; - backend.seed_object(bucket, &stale_metadata, b"{}".to_vec()).await; - - let stale_store = StrongTableCatalogStore::new(backend.clone()); - stale_store - .load_table(bucket, "sales", "orders") - .await - .expect("stale store should hydrate") - .expect("table should exist"); - - let first_result = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request-1".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata.clone(), - new_metadata_location: first_metadata.clone(), - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .expect("first commit should succeed"); - - let err = stale_store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-2".to_string(), - idempotency_key: Some("client-request-2".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: stale_metadata, - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .expect_err("stale snapshot CAS should fail"); - - assert_matches!(err, TableCatalogStoreError::Conflict(_)); - let loaded = stale_store - .load_table(bucket, "sales", "orders") - .await - .expect("stale store should reload after CAS conflict") - .expect("table should still exist"); - assert_eq!(loaded.metadata_location, first_result.table.metadata_location); - assert_eq!(loaded.version_token, first_result.table.version_token); - assert!( - stale_store - .get_commit_by_id(bucket, "table-id", "commit-2") - .await - .expect("commit lookup should succeed") - .is_none() - ); - assert!( - stale_store - .get_commit_by_idempotency_key(bucket, "table-id", "client-request-2") - .await - .expect("idempotency lookup should succeed") - .is_none() - ); - } - - #[tokio::test] - async fn strong_catalog_backing_refreshes_hydrated_reads_after_independent_commit() { - let backend = TestCatalogObjectBackend::default(); - let writer = StrongTableCatalogStore::new(backend.clone()); - let reader = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - writer - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - writer - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("namespace should be created"); - writer - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .expect("table should be created"); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - - let loaded_before_commit = reader - .load_table(bucket, "sales", "orders") - .await - .expect("reader should hydrate") - .expect("table should exist"); - assert_eq!(loaded_before_commit.metadata_location, current_metadata); - - let result = writer - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request-1".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .expect("writer commit should succeed"); - - let loaded_after_commit = reader - .load_table(bucket, "sales", "orders") - .await - .expect("reader should refresh durable state") - .expect("table should still exist"); - assert_eq!(loaded_after_commit.metadata_location, result.table.metadata_location); - assert_eq!(loaded_after_commit.version_token, result.table.version_token); - } - - #[tokio::test] - async fn strong_catalog_backing_skips_snapshot_body_when_etag_is_unchanged() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - - backend.reset_call_counts().await; - let loaded = store - .get_table_bucket(bucket) - .await - .expect("table bucket load should succeed") - .expect("table bucket should exist"); - - assert_eq!(loaded.table_bucket, bucket); - assert_eq!(backend.metadata_call_count().await, 1); - assert_eq!(backend.read_call_count().await, 0); - assert_eq!(backend.list_call_count().await, 0); - } - - #[tokio::test] - async fn strong_catalog_backing_resolves_data_plane_resource_without_catalog_scan() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("namespace should be created"); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata)) - .await - .expect("table should be created"); - - backend.reset_call_counts().await; - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/file.parquet") - .await - .expect("data-plane lookup should succeed") - .expect("data-plane resource should resolve"); - - assert_eq!(resource.table_bucket, bucket); - assert_eq!(resource.namespace, namespace.public_name()); - assert_eq!(resource.table, table.as_str()); - assert_eq!(resource.table_id, "table-id"); - assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); - assert_eq!(backend.metadata_call_count().await, 1); - assert_eq!(backend.read_call_count().await, 0); - assert_eq!(backend.list_call_count().await, 0); - } - - #[tokio::test] - async fn strong_catalog_backing_does_not_publish_warehouse_index_before_snapshot_persist() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); - - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect("namespace should be created"); - - let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; - let create_store = store.clone(); - let create_task = tokio::spawn(async move { - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let table = IdentifierSegment::parse("orders").expect("table should parse"); - create_store - .create_table(test_table_entry("analytics", &namespace, &table, current_metadata)) - .await - }); - pause.wait_started().await; - - let resource_before_commit = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/file.parquet") - .await - .expect("data-plane lookup should succeed while snapshot persist is paused"); - assert!(resource_before_commit.is_none()); - assert!( - store - .load_table(bucket, "sales", "orders") - .await - .expect("table load should succeed while snapshot persist is paused") - .is_none() - ); - - pause.release(); - create_task - .await - .expect("create task should join") - .expect("table should be created"); - - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/file.parquet") - .await - .expect("data-plane lookup should succeed after snapshot persist") - .expect("new table warehouse index should be visible after commit"); - assert_eq!(resource.table_bucket, bucket); - assert_eq!(resource.namespace, namespace.public_name()); - assert_eq!(resource.table, table.as_str()); - assert_eq!(resource.table_id, "table-id"); - assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); - } - - #[tokio::test] - async fn strong_catalog_backing_rejects_duplicate_snapshot_warehouse_index_entries() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let orders = IdentifierSegment::parse("orders").expect("table should parse"); - let returns = IdentifierSegment::parse("returns").expect("table should parse"); - let mut returns_entry = test_table_entry( - bucket, - &namespace, - &returns, - default_table_metadata_file_path(&namespace, &returns, "00001.metadata.json"), - ); - returns_entry.table_id = "table-id-2".to_string(); - returns_entry.table_uuid = "table-uuid-2".to_string(); - - let snapshot = StrongTableCatalogSnapshot { - version: STRONG_TABLE_CATALOG_SNAPSHOT_VERSION, - table_buckets: vec![test_bucket_entry(bucket)], - namespaces: vec![test_namespace_entry(bucket, &namespace)], - tables: vec![ - test_table_entry( - bucket, - &namespace, - &orders, - default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), - ), - returns_entry, - ], - views: Vec::new(), - commits: Vec::new(), - idempotency: Vec::new(), - }; - backend - .seed_object( - RUSTFS_META_BUCKET, - &StrongTableCatalogStore::::snapshot_object_path(), - serde_json::to_vec(&snapshot).expect("strong snapshot should encode"), - ) - .await; - - let err = store - .get_table_bucket(bucket) - .await - .expect_err("duplicate warehouse prefix should fail snapshot hydration"); - - assert_matches!(err, TableCatalogStoreError::Invalid(message) if message.contains("duplicate active table warehouse location")); - } - - #[tokio::test] - async fn strong_catalog_backing_does_not_publish_draft_when_persist_and_reload_fail() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").expect("namespace should parse"); - let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); - - store - .put_table_bucket(test_bucket_entry(bucket)) - .await - .expect("table bucket should be created"); - backend.fail_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; - backend.fail_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; - - let err = store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .expect_err("namespace write should fail"); - - assert_matches!(err, TableCatalogStoreError::Internal(_)); - let state = store.state.lock().await; - assert!( - !state - .namespaces - .contains_key(&StrongTableCatalogStore::::namespace_key(bucket, &namespace)) - ); - drop(state); - - assert!( - store - .list_namespaces(bucket) - .await - .expect("durable state should reload after transient failure") - .is_empty() - ); - } - - #[tokio::test] - async fn strong_catalog_backing_commit_conflict_keeps_pointer_and_wal_unchanged() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - - let err = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request".to_string()), - operation: "append".to_string(), - expected_version_token: "stale-token".to_string(), - expected_metadata_location: current_metadata.clone(), - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Conflict(_)); - let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(loaded.metadata_location, current_metadata); - assert_eq!(loaded.version_token, "token-v1"); - assert!( - store - .get_commit_by_id(bucket, "table-id", "commit-1") - .await - .unwrap() - .is_none() - ); - assert!( - store - .get_commit_by_idempotency_key(bucket, "table-id", "client-request") - .await - .unwrap() - .is_none() - ); - } - - #[tokio::test] - async fn strong_catalog_backing_rejects_invalid_table_warehouse_location() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - let mut entry = test_table_entry(bucket, &namespace, &table, current_metadata); - entry.warehouse_location = "s3://other-bucket/tables/table-id".to_string(); - - let err = store.create_table(entry).await.unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Invalid(_)); - assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_none()); - } - - #[tokio::test] - async fn strong_catalog_backing_rejects_duplicate_table_warehouse_location_on_register() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let orders = IdentifierSegment::parse("orders").unwrap(); - let customers = IdentifierSegment::parse("customers").unwrap(); - let orders_metadata = default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"); - let customers_metadata = default_table_metadata_file_path(&namespace, &customers, "00001.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &orders, orders_metadata)) - .await - .unwrap(); - let mut duplicate = test_table_entry(bucket, &namespace, &customers, customers_metadata); - duplicate.table_id = "table-id-2".to_string(); - duplicate.table_uuid = "table-uuid-2".to_string(); - duplicate.warehouse_location = format!("s3://{bucket}/tables/table-id"); - - let err = store.register_table(duplicate).await.unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Conflict(_)); - assert!(store.load_table(bucket, "sales", "customers").await.unwrap().is_none()); - } - - #[tokio::test] - async fn strong_catalog_backing_rejects_duplicate_table_warehouse_location_on_commit_relocation() { - let backend = TestCatalogObjectBackend::default(); - let store = StrongTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let orders = IdentifierSegment::parse("orders").unwrap(); - let customers = IdentifierSegment::parse("customers").unwrap(); - let orders_metadata = default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"); - let relocated_metadata = default_table_metadata_file_path(&namespace, &orders, "00002.metadata.json"); - let customers_metadata = default_table_metadata_file_path(&namespace, &customers, "00001.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &orders, orders_metadata.clone())) - .await - .unwrap(); - let mut customers_entry = test_table_entry(bucket, &namespace, &customers, customers_metadata); - customers_entry.table_id = "table-id-2".to_string(); - customers_entry.table_uuid = "table-uuid-2".to_string(); - customers_entry.warehouse_location = format!("s3://{bucket}/tables/customer-id"); - store.create_table(customers_entry).await.unwrap(); - backend - .seed_object( - bucket, - &relocated_metadata, - serde_json::to_vec(&serde_json::json!({ - "location": "s3://analytics/tables/customer-id", - "table-uuid": "table-uuid" - })) - .unwrap(), - ) - .await; - - let err = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: orders.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request".to_string()), - operation: "set-location".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: orders_metadata.clone(), - new_metadata_location: relocated_metadata, - requirements: Vec::new(), - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Conflict(_)); - let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(loaded.metadata_location, orders_metadata); - assert_eq!(loaded.warehouse_location, format!("s3://{bucket}/tables/table-id")); - assert!( - store - .get_commit_by_id(bucket, "table-id", "commit-1") - .await - .unwrap() - .is_none() - ); - } - - #[tokio::test] - async fn diagnostics_backing_manifest_requires_recovery_before_migration() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; - - store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap(); - - let diagnostics = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); - - assert_eq!(diagnostics.backing_manifest.current.wal.finalization_required_count, 1); - assert_eq!( - diagnostics.backing_manifest.migration.status, - TableCatalogBackingMigrationStatus::RecoveryRequired - ); - assert!( - diagnostics - .backing_manifest - .migration - .blockers - .contains(&TableCatalogBackingMigrationBlocker::CommitRecoveryRequired) - ); - } - - #[tokio::test] - async fn consistency_check_reports_missing_metadata_object() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - - let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); - - assert_eq!(report.catalog.table.metadata_location, current.clone()); - assert_eq!(report.current_metadata_status, TableMetadataPointerStatus::MissingObject); - assert_eq!(report.recovery_status, TableCatalogRecoveryStatus::ReadOnlyRecommended); - assert_eq!(report.recommended_actions, vec![TableCatalogRecoveryAction::RestoreCurrentMetadataObject]); - assert!(report.orphan_metadata_candidate_locations.is_empty()); - } - - #[tokio::test] - async fn consistency_check_reports_invalid_metadata_location() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let invalid_metadata = ".rustfs-table/warehouses/default/namespaces/sales/tables/other/metadata/00001.metadata.json"; - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, invalid_metadata.to_string())) - .await - .unwrap(); - - let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); - - assert_eq!(report.catalog.table.metadata_location, invalid_metadata); - assert_eq!(report.current_metadata_status, TableMetadataPointerStatus::InvalidLocation); - assert!(report.orphan_metadata_candidate_locations.is_empty()); - } - - #[tokio::test] - async fn orphan_metadata_scan_does_not_treat_largest_version_as_committed() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let uncommitted = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend - .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) - .await; - backend - .seed_object(bucket, &uncommitted, br#"{"metadata-log":[]}"#.to_vec()) - .await; - - let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); - - assert_eq!(report.current_metadata_status, TableMetadataPointerStatus::Valid); - assert_eq!(report.catalog.table.metadata_location, current); - assert_eq!(report.orphan_metadata_candidate_locations, vec![uncommitted]); - } - - #[tokio::test] - async fn orphan_metadata_scan_keeps_metadata_for_protected_snapshot_refs() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let orphan = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let tagged = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let current = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); - - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; - backend.seed_object(bucket, &orphan, b"{}".to_vec()).await; - backend - .seed_object( - bucket, - &tagged, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 10 - })) - .unwrap(), - ) - .await; - backend - .seed_object( - bucket, - ¤t, - serde_json::to_vec(&serde_json::json!({ - "current-snapshot-id": 20, - "metadata-log": [], - "refs": { - "audit": { - "snapshot-id": 10, - "type": "tag" - } - } - })) - .unwrap(), - ) - .await; - - let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); - - assert_eq!(report.current_metadata_status, TableMetadataPointerStatus::Valid); - assert_eq!(report.orphan_metadata_candidate_locations, vec![orphan]); - } - - #[tokio::test] - async fn object_table_catalog_store_commits_with_token_match_and_writes_log() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - - let result = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client/%2f\nrequest".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata.clone(), - requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap(); - - assert_eq!(result.table.metadata_location, new_metadata); - assert_ne!(result.table.version_token, "token-v1"); - assert_eq!(result.table.generation, 2); - assert_eq!(result.commit_log.status, CommitLogStatus::Committed); - - let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(loaded.metadata_location, result.table.metadata_location); - assert_eq!(loaded.version_token, result.table.version_token); - assert!( - store - .get_commit_by_id(bucket, "table-id", "commit-1") - .await - .unwrap() - .is_some() - ); - assert!( - store - .get_commit_by_idempotency_key(bucket, "table-id", "client/%2f\nrequest") - .await - .unwrap() - .is_some() - ); - } - - #[tokio::test] - async fn object_table_catalog_store_syncs_warehouse_location_from_committed_metadata() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - store.backfill_table_warehouse_index(bucket).await.unwrap(); - backend.reset_call_counts().await; - backend - .seed_object( - bucket, - &new_metadata, - serde_json::to_vec(&serde_json::json!({ - "location": "s3://analytics/tables/relocated-table-id", - "table-uuid": "table-uuid" - })) - .unwrap(), - ) - .await; - - let result = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "set-location".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata, - requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap(); - - assert_eq!(result.table.warehouse_location, "s3://analytics/tables/relocated-table-id"); - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/relocated-table-id/data/part-00001.parquet") - .await - .expect("data-plane resource lookup should succeed") - .expect("relocated table warehouse object should resolve to the table"); - assert_eq!(resource.table, "orders"); - assert_eq!(resource.warehouse_object_prefix, "tables/relocated-table-id/"); - let old_resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") - .await - .expect("old table warehouse lookup should succeed"); - assert!(old_resource.is_none()); - assert_eq!(backend.list_call_count().await, 0); - } - - #[tokio::test] - async fn object_table_catalog_store_reuses_old_prefix_after_failed_relocation_index_delete() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let next_table = IdentifierSegment::parse("returns").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let old_index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - let old_entry = test_table_entry(bucket, &namespace, &table, current_metadata.clone()); - store - .create_table(old_entry.clone()) - .await - .expect("old table should be created"); - backend - .seed_object( - bucket, - &new_metadata, - serde_json::to_vec(&serde_json::json!({ - "location": "s3://analytics/tables/relocated-table-id", - "table-uuid": "table-uuid" - })) - .unwrap(), - ) - .await; - backend.fail_delete_attempt(RUSTFS_META_BUCKET, &old_index_path, 1).await; - - store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "set-location".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata.clone(), - new_metadata_location: new_metadata, - requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], - writer: Some("pyiceberg/test".to_string()), - }) - .await - .unwrap(); - - let mut next_entry = test_table_entry(bucket, &namespace, &next_table, current_metadata); - next_entry.table_id = "next-table-id".to_string(); - next_entry.warehouse_location = format!("s3://{bucket}/tables/table-id"); - store - .create_table(next_entry) - .await - .expect("stale old warehouse index should not block prefix reuse"); - - let (index, _) = store - .read_entry::(store.catalog_bucket(), &old_index_path) - .await - .unwrap() - .expect("reused prefix index should exist"); - assert_eq!(index.table, "returns"); - assert_eq!(index.table_id, "next-table-id"); - - store - .delete_table_warehouse_index(&old_entry) - .await - .expect("old owner should not delete reused prefix index"); - let reused_resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") - .await - .expect("reused prefix lookup should succeed") - .expect("reused prefix should still resolve to the new table"); - assert_eq!(reused_resource.table, "returns"); - assert_eq!(reused_resource.table_id, "next-table-id"); - } - - #[tokio::test] - async fn object_table_catalog_store_does_not_advance_table_when_idempotency_staging_fails() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let idempotency_key = "client-request"; - let idempotency_path = - TableCatalogObjectPaths::default().commit_idempotency_entry_path(bucket, "table-id", idempotency_key); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - backend.fail_put_attempt(RUSTFS_META_BUCKET, &idempotency_path, 1).await; - - let err = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some(idempotency_key.to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata.clone(), - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Internal(_)); - let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(loaded.metadata_location, current_metadata); - assert_eq!(loaded.version_token, "token-v1"); - let staged = store.get_commit_by_id(bucket, "table-id", "commit-1").await.unwrap().unwrap(); - assert_eq!(staged.status, CommitLogStatus::Staged); - } - - #[tokio::test] - async fn object_table_catalog_store_recovers_staged_commit_after_post_cas_finalization_failure() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; - - let request = TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata.clone(), - requirements: Vec::new(), - writer: None, - }; - - let result = store.commit_table(request.clone()).await.unwrap(); - - assert_eq!(result.table.metadata_location, new_metadata); - assert_eq!(result.commit_log.status, CommitLogStatus::Committed); - let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(loaded.version_token, result.table.version_token); - let staged = store.get_commit_by_id(bucket, "table-id", "commit-1").await.unwrap().unwrap(); - assert_eq!(staged.status, CommitLogStatus::Staged); - - let retry = store.commit_table(request).await.unwrap(); - assert_eq!(retry.table.version_token, result.table.version_token); - assert_eq!(retry.commit_log.status, CommitLogStatus::Committed); - let committed = store.get_commit_by_id(bucket, "table-id", "commit-1").await.unwrap().unwrap(); - assert_eq!(committed.status, CommitLogStatus::Committed); - } - - #[tokio::test] - async fn table_commit_recovery_reports_post_cas_staged_commit() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; - - let result = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata.clone(), - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap(); - assert_eq!(result.commit_log.status, CommitLogStatus::Committed); - - let report = store.plan_table_commit_recovery(bucket, "sales", "orders").await.unwrap(); - - assert_eq!(report.current_metadata_location, new_metadata); - assert_eq!(report.finalization_required_count, 1); - assert_eq!(report.manual_review_count, 0); - assert_eq!(report.commits.len(), 1); - assert_eq!(report.commits[0].commit_id, "commit-1"); - assert_eq!(report.commits[0].status, CommitLogStatus::Staged); - assert_eq!(report.commits[0].recovery_state, TableCommitRecoveryState::FinalizationRequired); - } - - #[tokio::test] - async fn diagnostics_report_includes_table_commit_recovery_state() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; - - store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap(); - - let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); - - assert_eq!(report.recovery_status, TableCatalogRecoveryStatus::Recoverable); - assert_eq!(report.recommended_actions, vec![TableCatalogRecoveryAction::RunCommitRecovery]); - assert_eq!(report.commit_recovery.finalization_required_count, 1); - assert_eq!(report.commit_recovery.commits.len(), 1); - assert_eq!( - report.commit_recovery.commits[0].recovery_state, - TableCommitRecoveryState::FinalizationRequired - ); - } - - #[tokio::test] - async fn table_commit_recovery_finalizes_post_cas_staged_commit() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; - - store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap(); - - let report = store.recover_table_commits(bucket, "sales", "orders").await.unwrap(); - - assert_eq!(report.finalized_count, 1); - assert_eq!(report.finalization_required_count, 0); - let committed = store.get_commit_by_id(bucket, "table-id", "commit-1").await.unwrap().unwrap(); - assert_eq!(committed.status, CommitLogStatus::Committed); - } - - #[tokio::test] - async fn table_commit_recovery_reports_staged_commit_after_table_cas_failure() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let table_path = TableCatalogObjectPaths::default().table_entry_path(bucket, &namespace, &table); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, ¤t_metadata, b"{}".to_vec()).await; - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - backend.fail_put_attempt(RUSTFS_META_BUCKET, &table_path, 2).await; - - let err = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata.clone(), - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap_err(); - assert_matches!(err, TableCatalogStoreError::Internal(_)); - let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(loaded.metadata_location, current_metadata); - - let report = store.plan_table_commit_recovery(bucket, "sales", "orders").await.unwrap(); - assert_eq!(report.staged_before_table_update_count, 1); - assert_eq!(report.finalization_required_count, 0); - assert_eq!(report.commits[0].recovery_state, TableCommitRecoveryState::StagedBeforeTableUpdate); - - let diagnostics = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); - assert_eq!(diagnostics.current_metadata_status, TableMetadataPointerStatus::Valid); - assert_eq!(diagnostics.recovery_status, TableCatalogRecoveryStatus::Recoverable); - assert_eq!(diagnostics.recommended_actions, vec![TableCatalogRecoveryAction::RetryCommit]); - } - - #[tokio::test] - async fn table_commit_recovery_repairs_stale_idempotency_index_after_partial_finalization() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - let idempotency_path = - TableCatalogObjectPaths::default().commit_idempotency_entry_path(bucket, "table-id", "client-request-1"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - backend.fail_put_attempt(RUSTFS_META_BUCKET, &idempotency_path, 2).await; - - let result = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: Some("client-request-1".to_string()), - operation: "append".to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: current_metadata, - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap(); - assert_eq!(result.commit_log.status, CommitLogStatus::Committed); - let stale_index = store - .get_commit_by_idempotency_key(bucket, "table-id", "client-request-1") - .await - .unwrap() - .unwrap(); - assert_eq!(stale_index.status, CommitLogStatus::Staged); - - let report = store.plan_table_commit_recovery(bucket, "sales", "orders").await.unwrap(); - assert_eq!(report.idempotency_repair_required_count, 1); - assert_eq!(report.manual_review_count, 0); - assert_eq!(report.commits[0].recovery_state, TableCommitRecoveryState::IdempotencyIndexRepairRequired); - assert_eq!(report.commits[0].idempotency_index_status, TableCommitIdempotencyIndexStatus::Stale); - - let repaired = store.recover_table_commits(bucket, "sales", "orders").await.unwrap(); - - assert_eq!(repaired.finalized_count, 1); - assert_eq!(repaired.idempotency_repair_required_count, 0); - let repaired_index = store - .get_commit_by_idempotency_key(bucket, "table-id", "client-request-1") - .await - .unwrap() - .unwrap(); - assert_eq!(repaired_index.status, CommitLogStatus::Committed); - } - - #[tokio::test] - async fn object_table_catalog_store_rejects_stale_commit_token() { - let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); - let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); - let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); - - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); - store - .create_namespace(test_namespace_entry(bucket, &namespace)) - .await - .unwrap(); - store - .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) - .await - .unwrap(); - backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; - - let err = store - .commit_table(TableCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: "commit-1".to_string(), - idempotency_key: None, - operation: "append".to_string(), - expected_version_token: "stale-token".to_string(), - expected_metadata_location: current_metadata.clone(), - new_metadata_location: new_metadata, - requirements: Vec::new(), - writer: None, - }) - .await - .unwrap_err(); - - assert_matches!(err, TableCatalogStoreError::Conflict(_)); - let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); - assert_eq!(loaded.metadata_location, current_metadata); - assert_eq!(loaded.version_token, "token-v1"); - assert!( - store - .get_commit_by_id(bucket, "table-id", "commit-1") - .await - .unwrap() - .is_none() - ); - } - - #[test] - fn namespace_marker_path_stays_under_default_reserved_boundary() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - - assert_eq!( - default_namespace_marker_path(&namespace), - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/namespace.json" - ); - assert_eq!(namespace.public_name(), "analytics.daily_events"); - } - - #[test] - fn namespace_marker_path_extracts_public_name() { - assert_eq!( - namespace_name_from_marker_path(".rustfs-table/warehouses/default/namespaces/analytics/daily_events/namespace.json"), - Some("analytics.daily_events".to_string()) - ); - assert_eq!( - namespace_name_from_marker_path( - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/current.json" - ), - None - ); - assert_eq!( - namespace_name_from_marker_path(".rustfs-table/warehouses/other/namespaces/analytics/daily_events/namespace.json"), - None - ); - } - - #[test] - fn namespace_marker_json_uses_stable_catalog_defaults() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - let marker = serde_json::to_value(NamespaceMarker::new(&namespace)).unwrap(); - - assert_eq!(marker["version"], TABLE_NAMESPACE_MARKER_VERSION); - assert_eq!(marker["namespace"], "analytics.daily_events"); - assert!(!namespace_marker_json(&namespace).unwrap().is_empty()); - } - - #[test] - fn table_marker_path_stays_under_namespace_reserved_boundary() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - let table = IdentifierSegment::parse("events").unwrap(); - - assert_eq!( - default_table_root_prefix(&namespace), - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/" - ); - assert_eq!( - default_table_marker_path(&namespace, &table), - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/table.json" - ); - } - - #[test] - fn table_marker_path_extracts_table_name() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - - assert_eq!( - table_name_from_marker_path( - &namespace, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/table.json" - ), - Some("events".to_string()) - ); - assert_eq!( - table_name_from_marker_path( - &namespace, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/current.json" - ), - None - ); - assert_eq!( - table_name_from_marker_path( - &namespace, - ".rustfs-table/warehouses/default/namespaces/analytics/other/tables/events/table.json" - ), - None - ); - } - - #[test] - fn table_marker_json_uses_stable_catalog_defaults() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - let table = IdentifierSegment::parse("events").unwrap(); - let marker = serde_json::to_value(TableMarker::new(&namespace, &table)).unwrap(); - - assert_eq!(marker["version"], TABLE_RESOURCE_MARKER_VERSION); - assert_eq!(marker["namespace"], "analytics.daily_events"); - assert_eq!(marker["name"], "events"); - assert!(marker["metadata_location"].is_null()); - assert!(!table_marker_json(&namespace, &table).unwrap().is_empty()); - } - - #[test] - fn table_current_pointer_path_stays_under_table_boundary() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - let table = IdentifierSegment::parse("events").unwrap(); - - assert_eq!( - default_table_metadata_dir_path(&namespace, &table), - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata" - ); - assert_eq!( - default_table_current_pointer_path(&namespace, &table), - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/current.json" - ); - assert_eq!( - default_table_lifecycle_path(&namespace, &table), - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/lifecycle.json" - ); - } - - #[test] - fn table_metadata_file_path_stays_under_metadata_boundary() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - let table = IdentifierSegment::parse("events").unwrap(); - let table_identifier = - TableIdentifier::new(IdentifierSegment::parse(DEFAULT_WAREHOUSE_ID).unwrap(), namespace.clone(), table.clone()); - - assert_eq!( - default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" - ); - assert_eq!( - TablePathResolver::default().metadata_file_path(&table_identifier, "00001.metadata.json"), - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" - ); - } - - #[test] - fn table_metadata_file_name_validation_rejects_unsafe_names() { - assert!(is_valid_table_metadata_file_name("00001.metadata.json")); - assert!(is_valid_table_metadata_file_name("v1-4f2c_metadata.json")); - - assert!(!is_valid_table_metadata_file_name("")); - assert!(!is_valid_table_metadata_file_name(".metadata.json")); - assert!(!is_valid_table_metadata_file_name("00001.metadata")); - assert!(!is_valid_table_metadata_file_name("00001.JSON")); - assert!(!is_valid_table_metadata_file_name("../current.json")); - assert!(!is_valid_table_metadata_file_name("nested/00001.json")); - assert!(!is_valid_table_metadata_file_name("nested%2f00001.json")); - assert!(!is_valid_table_metadata_file_name("00001\\metadata.json")); - assert!(!is_valid_table_metadata_file_name("00001\nmetadata.json")); - } - - #[test] - fn table_metadata_location_validation_stays_inside_metadata_dir() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - let table = IdentifierSegment::parse("events").unwrap(); - - assert!(is_valid_table_metadata_location( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" - )); - assert!(!is_valid_table_metadata_location(&namespace, &table, "")); - assert!(!is_valid_table_metadata_location( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/current.json" - )); - assert!(!is_valid_table_metadata_location( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/other/metadata/00001.json" - )); - assert!(!is_valid_table_metadata_location( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/../current.json" - )); - assert!(!is_valid_table_metadata_location( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/nested/00001.json" - )); - } - - #[test] - fn metadata_location_from_metadata_file_path_extracts_table_metadata_only() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - let table = IdentifierSegment::parse("events").unwrap(); - - assert_eq!( - metadata_location_from_metadata_file_path( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" - ), - Some( - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" - .to_string() - ) - ); - assert_eq!( - metadata_location_from_metadata_file_path( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/current.json" - ), - None - ); - assert_eq!( - metadata_location_from_metadata_file_path( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/nested/00001.metadata.json" - ), - None - ); - assert_eq!( - metadata_location_from_metadata_file_path( - &namespace, - &table, - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/other/metadata/00001.metadata.json" - ), - None - ); - } - - #[test] - fn table_metadata_pointer_json_round_trips() { - let location = - ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.json".to_string(); - let data = table_metadata_pointer_json(location.clone()).unwrap(); - let pointer = parse_table_metadata_pointer(&data).unwrap(); - - assert_eq!(pointer.version, TABLE_METADATA_POINTER_VERSION); - assert_eq!(pointer.metadata_location, location); - } - - #[test] - fn object_mutation_entrypoints_call_reserved_prefix_guard() { - let source = include_str!("app/object_usecase.rs"); - let delete_object = source - .split_once("pub async fn execute_delete_object") - .and_then(|(_, remainder)| remainder.split_once("pub async fn execute_head_object")) - .map(|(delete_object, _)| delete_object) - .expect("delete object entrypoint should remain in the object usecase"); - - for expected in [ - "validate_object_key(&key, request_method_name)?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", - "validate_object_key(&key, \"COPY (dest)\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", - "if let Err(err) = validate_table_catalog_object_mutation(&bucket, &obj_id.key).await", - "validate_table_catalog_object_mutation(&bucket, &object).await?;", - "validate_object_key(&key, \"PUT\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", - "validate_table_catalog_object_mutation(&bucket, &fpath).await?;", - ] { - assert!(source.contains(expected), "missing object mutation guard: {expected}"); - } - assert!( - delete_object.contains("validate_object_key(&key, \"DELETE\")?;") - && delete_object.contains("validate_table_catalog_object_mutation(&bucket, &key).await?;"), - "delete object entrypoint must validate the object key and reserved catalog prefix" - ); - } - - #[test] - fn multipart_mutation_entrypoints_call_reserved_prefix_guard() { - let source = include_str!("app/multipart_usecase.rs"); - - assert_eq!( - source - .matches("validate_table_catalog_object_mutation(&bucket, &key).await?;") - .count(), - 4 - ); - } - - #[test] - fn object_metadata_mutation_entrypoints_call_reserved_prefix_guard() { - let source = include_str!("storage/ecfs.rs"); - - assert_eq!( - source - .matches("validate_table_catalog_object_mutation(&bucket, &object).await?;") - .count(), - 2 - ); - assert_eq!( - source - .matches("validate_table_catalog_object_mutation(&bucket, &key).await?;") - .count(), - 2 - ); - } - - #[test] - fn identifier_segment_accepts_conservative_catalog_names() { - for value in [ - "a", - "a1", - "a-b", - "a_b", - "abc123", - "a23456789012345678901234567890123456789012345678901234567890123", - ] { - assert_eq!(IdentifierSegment::parse(value).unwrap().as_str(), value); - } - } - - #[test] - fn identifier_segment_rejects_ambiguous_or_unsafe_names() { - for value in [ - "", - ".", - "..", - "Upper", - "has.dot", - "has/slash", - "has\\slash", - "has%2fslash", - "-leading", - "trailing-", - "_leading", - "trailing_", - "has space", - "name\nbreak", - ] { - assert!(IdentifierSegment::parse(value).is_err(), "value should be rejected: {value:?}"); - } - - let too_long = "a".repeat(IdentifierSegment::MAX_LEN + 1); - assert!(IdentifierSegment::parse(too_long).is_err()); - } - - #[test] - fn namespace_uses_dot_syntax_for_public_identity_and_slash_for_storage() { - let namespace = Namespace::parse("analytics.daily_events").unwrap(); - - assert_eq!(namespace.segments().len(), 2); - assert_eq!(namespace.storage_id(), "analytics/daily_events"); - } - - #[test] - fn namespace_length_is_bounded_for_catalog_paths_and_page_tokens() { - let mut segments = vec!["a".repeat(63); 8]; - segments[0].push('a'); - let max_length_namespace = segments.join("."); - assert_eq!(max_length_namespace.len(), Namespace::MAX_LEN); - Namespace::parse(&max_length_namespace).expect("namespace at the maximum length should parse"); - - let namespace = format!("{max_length_namespace}.a"); - assert_eq!( - Namespace::parse(&namespace), - Err(CatalogIdentifierError::NamespaceTooLong { max: Namespace::MAX_LEN }) - ); - } - - #[test] - fn resolver_builds_paths_under_reserved_table_boundary() { - let table = TableIdentifier::new( - IdentifierSegment::parse("warehouse1").unwrap(), - Namespace::parse("analytics.daily").unwrap(), - IdentifierSegment::parse("events").unwrap(), - ); - let resolver = TablePathResolver::default(); - - assert_eq!( - resolver.current_pointer_path(&table), - ".rustfs-table/warehouses/warehouse1/namespaces/analytics/daily/tables/events/current.json" - ); - assert_eq!( - resolver.metadata_dir_path(&table), - ".rustfs-table/warehouses/warehouse1/namespaces/analytics/daily/tables/events/metadata" - ); - } -} diff --git a/rustfs/src/table_catalog/error.rs b/rustfs/src/table_catalog/error.rs new file mode 100644 index 000000000..9b2fef4e1 --- /dev/null +++ b/rustfs/src/table_catalog/error.rs @@ -0,0 +1,79 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CatalogIdentifierError { + Empty, + TooLong { max: usize }, + NamespaceTooLong { max: usize }, + InvalidCharacter, + InvalidBoundary, + Ambiguous, +} + +impl fmt::Display for CatalogIdentifierError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => f.write_str("catalog identifier segment is empty"), + Self::TooLong { max } => write!(f, "catalog identifier segment exceeds {max} characters"), + Self::NamespaceTooLong { max } => write!(f, "catalog namespace exceeds {max} characters"), + Self::InvalidCharacter => f.write_str("catalog identifier segment contains invalid characters"), + Self::InvalidBoundary => { + f.write_str("catalog identifier segment must start and end with a lowercase letter or digit") + } + Self::Ambiguous => f.write_str("catalog identifier segment is ambiguous"), + } + } +} + +impl std::error::Error for CatalogIdentifierError {} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TableObjectMutationError { + ReservedCatalogObject, +} + +impl fmt::Display for TableObjectMutationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ReservedCatalogObject => f.write_str("object key is reserved for the table catalog"), + } + } +} + +impl std::error::Error for TableObjectMutationError {} +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum TableCatalogStoreError { + NotFound(String), + Conflict(String), + Invalid(String), + Internal(String), +} + +impl fmt::Display for TableCatalogStoreError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotFound(message) => write!(f, "table catalog entry not found: {message}"), + Self::Conflict(message) => write!(f, "table catalog conflict: {message}"), + Self::Invalid(message) => write!(f, "invalid table catalog entry: {message}"), + Self::Internal(message) => write!(f, "table catalog store error: {message}"), + } + } +} + +impl std::error::Error for TableCatalogStoreError {} + +pub(crate) type TableCatalogStoreResult = Result; diff --git a/rustfs/src/table_catalog/iceberg/commit.rs b/rustfs/src/table_catalog/iceberg/commit.rs new file mode 100644 index 000000000..1bc90bb13 --- /dev/null +++ b/rustfs/src/table_catalog/iceberg/commit.rs @@ -0,0 +1,303 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::*; + +pub(crate) fn commit_log_matches_request(commit_log: &CommitLogEntry, request: &TableCommitRequest, table_id: &str) -> bool { + commit_log.version == TABLE_CATALOG_ENTRY_VERSION + && commit_log.commit_id == request.commit_id + && commit_log.idempotency_key == request.idempotency_key + && commit_log.table_id == table_id + && commit_log.operation == request.operation + && commit_log.expected_version_token == request.expected_version_token + && commit_log.previous_metadata_location == request.expected_metadata_location + && commit_log.new_metadata_location == request.new_metadata_location + && commit_log.requirements == request.requirements + && commit_log.writer == request.writer +} + +pub(crate) fn table_matches_committed_log(table: &TableEntry, commit_log: &CommitLogEntry) -> bool { + table.table_id == commit_log.table_id + && table.metadata_location == commit_log.new_metadata_location + && table.version_token == commit_log.new_version_token +} + +pub(crate) fn table_matches_staged_base(table: &TableEntry, commit_log: &CommitLogEntry) -> bool { + table.table_id == commit_log.table_id + && table.metadata_location == commit_log.previous_metadata_location + && table.version_token == commit_log.expected_version_token +} + +pub(crate) fn table_catalog_recovery_summary( + metadata_status: &TableMetadataPointerStatus, + commit_recovery: &TableCommitRecoveryReport, +) -> (TableCatalogRecoveryStatus, Vec) { + let mut actions = Vec::new(); + let metadata_status = match metadata_status { + TableMetadataPointerStatus::Valid => None, + TableMetadataPointerStatus::MissingObject => { + actions.push(TableCatalogRecoveryAction::RestoreCurrentMetadataObject); + Some(TableCatalogRecoveryStatus::ReadOnlyRecommended) + } + TableMetadataPointerStatus::InvalidJson => { + actions.push(TableCatalogRecoveryAction::FixCurrentMetadataJson); + Some(TableCatalogRecoveryStatus::ReadOnlyRecommended) + } + TableMetadataPointerStatus::InvalidLocation => { + actions.push(TableCatalogRecoveryAction::MoveCurrentMetadataInsideTable); + Some(TableCatalogRecoveryStatus::ReadOnlyRecommended) + } + }; + + if commit_recovery.manual_review_count > 0 { + actions.push(TableCatalogRecoveryAction::ReviewCommitLog); + return (metadata_status.unwrap_or(TableCatalogRecoveryStatus::ManualReviewRequired), actions); + } + if commit_recovery.finalization_required_count > 0 || commit_recovery.idempotency_repair_required_count > 0 { + actions.push(TableCatalogRecoveryAction::RunCommitRecovery); + return (metadata_status.unwrap_or(TableCatalogRecoveryStatus::Recoverable), actions); + } + if commit_recovery.staged_before_table_update_count > 0 { + actions.push(TableCatalogRecoveryAction::RetryCommit); + return (metadata_status.unwrap_or(TableCatalogRecoveryStatus::Recoverable), actions); + } + + (metadata_status.unwrap_or(TableCatalogRecoveryStatus::Healthy), actions) +} + +fn commit_logs_share_recovery_payload(left: &CommitLogEntry, right: &CommitLogEntry) -> bool { + left.version == right.version + && left.commit_id == right.commit_id + && left.idempotency_key == right.idempotency_key + && left.table_id == right.table_id + && left.operation == right.operation + && left.expected_version_token == right.expected_version_token + && left.new_version_token == right.new_version_token + && left.previous_metadata_location == right.previous_metadata_location + && left.new_metadata_location == right.new_metadata_location + && left.requirements == right.requirements + && left.writer == right.writer +} + +fn commit_idempotency_index_status( + commit_log: &CommitLogEntry, + idempotency_commit: Option<&CommitLogEntry>, +) -> TableCommitIdempotencyIndexStatus { + match (commit_log.idempotency_key.as_ref(), idempotency_commit) { + (None, _) => TableCommitIdempotencyIndexStatus::NotRequired, + (Some(_), None) => TableCommitIdempotencyIndexStatus::Missing, + (Some(_), Some(indexed)) if indexed == commit_log => TableCommitIdempotencyIndexStatus::Matches, + (Some(_), Some(indexed)) if commit_logs_share_recovery_payload(indexed, commit_log) => { + TableCommitIdempotencyIndexStatus::Stale + } + (Some(_), Some(_)) => TableCommitIdempotencyIndexStatus::Conflicting, + } +} + +pub(crate) fn table_commit_recovery_entry( + table: &TableEntry, + commit_log: &CommitLogEntry, + idempotency_commit: Option<&CommitLogEntry>, +) -> TableCommitRecoveryEntry { + let idempotency_index_status = commit_idempotency_index_status(commit_log, idempotency_commit); + let idempotency_index_present = matches!( + idempotency_index_status, + TableCommitIdempotencyIndexStatus::Matches + | TableCommitIdempotencyIndexStatus::Stale + | TableCommitIdempotencyIndexStatus::Conflicting + ); + let idempotency_index_repair_required = matches!( + idempotency_index_status, + TableCommitIdempotencyIndexStatus::Missing | TableCommitIdempotencyIndexStatus::Stale + ); + + let (recovery_state, reason) = if matches!(idempotency_index_status, TableCommitIdempotencyIndexStatus::Conflicting) { + ( + TableCommitRecoveryState::ManualReview, + "idempotency index points at a different commit payload".to_string(), + ) + } else if table_matches_committed_log(table, commit_log) { + if matches!(commit_log.status, CommitLogStatus::Committed) { + if idempotency_index_repair_required { + ( + TableCommitRecoveryState::IdempotencyIndexRepairRequired, + "committed table pointer is durable but idempotency index needs repair".to_string(), + ) + } else { + ( + TableCommitRecoveryState::Committed, + "commit log and current table pointer agree".to_string(), + ) + } + } else { + ( + TableCommitRecoveryState::FinalizationRequired, + "current table pointer already advanced but commit log is not finalized".to_string(), + ) + } + } else if matches!(commit_log.status, CommitLogStatus::Committed) { + if idempotency_index_repair_required { + ( + TableCommitRecoveryState::IdempotencyIndexRepairRequired, + "historical committed log needs idempotency index repair".to_string(), + ) + } else { + ( + TableCommitRecoveryState::Committed, + "commit is finalized and may be older than the current table pointer".to_string(), + ) + } + } else if table_matches_staged_base(table, commit_log) { + ( + TableCommitRecoveryState::StagedBeforeTableUpdate, + "staged commit exists but table pointer has not advanced".to_string(), + ) + } else { + ( + TableCommitRecoveryState::ManualReview, + "staged commit no longer matches the current table pointer or its expected base".to_string(), + ) + }; + + TableCommitRecoveryEntry { + commit_id: commit_log.commit_id.clone(), + idempotency_key: commit_log.idempotency_key.clone(), + operation: commit_log.operation.clone(), + status: commit_log.status.clone(), + recovery_state, + previous_metadata_location: commit_log.previous_metadata_location.clone(), + new_metadata_location: commit_log.new_metadata_location.clone(), + expected_version_token: commit_log.expected_version_token.clone(), + new_version_token: commit_log.new_version_token.clone(), + idempotency_index_present, + idempotency_index_status, + reason, + } +} + +pub(crate) fn record_table_commit_attempt(operation: &str) { + counter!("rustfs_table_catalog_commit_attempts_total", "operation" => operation.to_string()).increment(1); +} + +fn table_catalog_store_result_label(result: &TableCatalogStoreResult) -> &'static str { + match result { + Ok(_) => "success", + Err(TableCatalogStoreError::Conflict(_)) => "conflict", + Err(TableCatalogStoreError::Invalid(_)) => "invalid", + Err(TableCatalogStoreError::NotFound(_)) => "not_found", + Err(TableCatalogStoreError::Internal(_)) => "failure", + } +} + +fn duration_millis_u64(duration: StdDuration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +pub(crate) fn record_table_commit_cas_result(operation: &str, started: Instant, result: &TableCatalogStoreResult<()>) { + let elapsed = started.elapsed(); + let result_label = table_catalog_store_result_label(result); + counter!( + "rustfs_table_catalog_commit_cas_results_total", + "operation" => operation.to_string(), + "result" => result_label.to_string() + ) + .increment(1); + histogram!( + "rustfs_table_catalog_commit_cas_duration_seconds", + "operation" => operation.to_string(), + "result" => result_label.to_string() + ) + .record(elapsed.as_secs_f64()); +} + +fn record_table_commit_result( + table_bucket: &str, + namespace: &str, + table: &str, + commit_id: &str, + operation: &str, + started: Instant, + result: &TableCatalogStoreResult, +) { + let elapsed = started.elapsed(); + let result_label = table_catalog_store_result_label(result); + counter!( + "rustfs_table_catalog_commit_results_total", + "operation" => operation.to_string(), + "result" => result_label.to_string() + ) + .increment(1); + if matches!(result, Err(TableCatalogStoreError::Conflict(_))) { + counter!("rustfs_table_catalog_commit_conflicts_total", "operation" => operation.to_string()).increment(1); + } + histogram!( + "rustfs_table_catalog_commit_duration_seconds", + "operation" => operation.to_string(), + "result" => result_label.to_string() + ) + .record(elapsed.as_secs_f64()); + + match result { + Ok(commit) if elapsed >= TABLE_COMMIT_SLOW_LOG_THRESHOLD => { + tracing::warn!( + table_bucket, + namespace, + table, + commit_id, + operation, + generation = commit.table.generation, + duration_ms = duration_millis_u64(elapsed), + "slow table catalog commit" + ); + } + Ok(commit) => { + tracing::debug!( + table_bucket, + namespace, + table, + commit_id, + operation, + generation = commit.table.generation, + duration_ms = duration_millis_u64(elapsed), + "table catalog commit completed" + ); + } + Err(error) => { + tracing::warn!( + table_bucket, + namespace, + table, + commit_id, + operation, + result = result_label, + duration_ms = duration_millis_u64(elapsed), + error = %error, + "table catalog commit did not complete" + ); + } + } +} + +pub(crate) fn table_commit_result( + table_bucket: &str, + namespace: &str, + table: &str, + commit_id: &str, + operation: &str, + started: Instant, + result: TableCatalogStoreResult, +) -> TableCatalogStoreResult { + record_table_commit_result(table_bucket, namespace, table, commit_id, operation, started, &result); + result +} diff --git a/rustfs/src/table_catalog/iceberg/manifest.rs b/rustfs/src/table_catalog/iceberg/manifest.rs new file mode 100644 index 000000000..933697bfe --- /dev/null +++ b/rustfs/src/table_catalog/iceberg/manifest.rs @@ -0,0 +1,177 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::*; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ManifestDataFileReference { + pub location: String, + pub content: ManifestDataFileContent, + pub object_kind: TableMetadataMaintenanceObjectKind, + pub entry_status: Option, + pub snapshot_id: Option, + pub sequence_number: Option, + pub file_sequence_number: Option, + pub record_count: Option, + pub file_size_bytes: Option, + pub partition: Vec<(String, apache_avro::types::Value)>, + pub sort_order_id: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ManifestDataFileContent { + Data, + PositionDelete, + EqualityDelete, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ManifestListReference { + pub manifest_path: String, + pub partition_spec_id: Option, + pub sequence_number: Option, + pub added_snapshot_id: Option, +} + +pub(crate) fn manifest_paths_from_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult> { + Ok(manifest_list_references_from_manifest_list_avro(data)? + .into_iter() + .map(|reference| reference.manifest_path) + .collect()) +} + +pub(crate) fn manifest_list_references_from_manifest_list_avro( + data: &[u8], +) -> TableCatalogStoreResult> { + let reader = apache_avro::Reader::new(data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest list Avro: {err}")))?; + let mut manifest_paths = Vec::new(); + for value in reader { + let value = + value.map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest list record: {err}")))?; + let manifest_path = avro_record_field(&value, "manifest_path") + .and_then(avro_string_value) + .ok_or_else(|| TableCatalogStoreError::Invalid("manifest list entry missing manifest_path".to_string()))?; + manifest_paths.push(ManifestListReference { + manifest_path: manifest_path.to_string(), + partition_spec_id: avro_record_field(&value, "partition_spec_id").and_then(avro_i32_value), + sequence_number: avro_record_field(&value, "sequence_number").and_then(avro_i64_value), + added_snapshot_id: avro_record_field(&value, "added_snapshot_id").and_then(avro_i64_value), + }); + } + Ok(manifest_paths) +} + +pub(crate) fn file_references_from_manifest_avro( + data: &[u8], +) -> TableCatalogStoreResult> { + Ok(data_file_references_from_manifest_avro(data)? + .into_iter() + .map(|reference| (reference.location, reference.object_kind)) + .collect()) +} + +pub(crate) fn data_file_references_from_manifest_avro(data: &[u8]) -> TableCatalogStoreResult> { + let reader = apache_avro::Reader::new(data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest Avro: {err}")))?; + let mut files = Vec::new(); + for value in reader { + let value = value.map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest record: {err}")))?; + let data_file = avro_record_field(&value, "data_file") + .ok_or_else(|| TableCatalogStoreError::Invalid("manifest entry missing data_file".to_string()))?; + let file_path = avro_record_field(data_file, "file_path") + .and_then(avro_string_value) + .ok_or_else(|| TableCatalogStoreError::Invalid("manifest data file missing file_path".to_string()))?; + let content = avro_record_field(data_file, "content") + .and_then(avro_i32_value) + .ok_or_else(|| TableCatalogStoreError::Invalid("manifest data file missing content".to_string()))?; + let (content, object_kind) = match content { + 0 => (ManifestDataFileContent::Data, TableMetadataMaintenanceObjectKind::DataFile), + 1 => (ManifestDataFileContent::PositionDelete, TableMetadataMaintenanceObjectKind::DeleteFile), + 2 => (ManifestDataFileContent::EqualityDelete, TableMetadataMaintenanceObjectKind::DeleteFile), + _ => continue, + }; + files.push(ManifestDataFileReference { + location: file_path.to_string(), + content, + object_kind, + entry_status: avro_record_field(&value, "status").and_then(avro_i32_value), + snapshot_id: avro_record_field(&value, "snapshot_id").and_then(avro_i64_value), + sequence_number: avro_record_field(&value, "sequence_number").and_then(avro_i64_value), + file_sequence_number: avro_record_field(&value, "file_sequence_number").and_then(avro_i64_value), + record_count: avro_record_field(data_file, "record_count") + .and_then(avro_i64_value) + .and_then(|value| u64::try_from(value).ok()), + file_size_bytes: avro_record_field(data_file, "file_size_in_bytes") + .and_then(avro_i64_value) + .and_then(|value| u64::try_from(value).ok()), + partition: avro_record_field(data_file, "partition") + .and_then(avro_record_value_fields) + .unwrap_or_default(), + sort_order_id: avro_record_field(data_file, "sort_order_id").and_then(avro_i32_value), + }); + } + Ok(files) +} + +fn avro_record_field<'a>(value: &'a apache_avro::types::Value, name: &str) -> Option<&'a apache_avro::types::Value> { + let value = avro_non_union_value(value); + let apache_avro::types::Value::Record(fields) = value else { + return None; + }; + fields + .iter() + .find_map(|(field_name, field_value)| (field_name == name).then_some(avro_non_union_value(field_value))) +} + +fn avro_record_value_fields(value: &apache_avro::types::Value) -> Option> { + let value = avro_non_union_value(value); + let apache_avro::types::Value::Record(fields) = value else { + return None; + }; + Some( + fields + .iter() + .map(|(field_name, field_value)| (field_name.clone(), avro_non_union_value(field_value).clone())) + .collect(), + ) +} + +pub(crate) fn avro_non_union_value(value: &apache_avro::types::Value) -> &apache_avro::types::Value { + match value { + apache_avro::types::Value::Union(_, inner) => avro_non_union_value(inner), + value => value, + } +} + +fn avro_string_value(value: &apache_avro::types::Value) -> Option<&str> { + match avro_non_union_value(value) { + apache_avro::types::Value::String(value) => Some(value), + _ => None, + } +} + +fn avro_i32_value(value: &apache_avro::types::Value) -> Option { + match avro_non_union_value(value) { + apache_avro::types::Value::Int(value) => Some(*value), + _ => None, + } +} + +fn avro_i64_value(value: &apache_avro::types::Value) -> Option { + match avro_non_union_value(value) { + apache_avro::types::Value::Long(value) => Some(*value), + _ => None, + } +} diff --git a/rustfs/src/table_catalog/iceberg/metadata.rs b/rustfs/src/table_catalog/iceberg/metadata.rs new file mode 100644 index 000000000..fdeab33f7 --- /dev/null +++ b/rustfs/src/table_catalog/iceberg/metadata.rs @@ -0,0 +1,120 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::*; + +pub(crate) fn metadata_log_locations( + current_metadata: &serde_json::Value, + namespace: &Namespace, + table: &IdentifierSegment, +) -> BTreeSet { + let mut locations = BTreeSet::new(); + let Some(metadata_log) = current_metadata.get("metadata-log").and_then(serde_json::Value::as_array) else { + return locations; + }; + + for entry in metadata_log { + let Some(metadata_location) = entry.get("metadata-file").and_then(serde_json::Value::as_str) else { + continue; + }; + if is_valid_table_metadata_location(namespace, table, metadata_location) { + locations.insert(metadata_location.to_string()); + } + } + + locations +} + +pub(crate) async fn metadata_locations_for_protected_snapshot_refs( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + current_metadata: &serde_json::Value, + metadata_locations: &[String], +) -> TableCatalogStoreResult> +where + B: TableCatalogObjectBackend, +{ + let protected_snapshot_ids = protected_ref_snapshot_ids(current_metadata); + if protected_snapshot_ids.is_empty() { + return Ok(BTreeSet::new()); + } + + let mut retained = BTreeSet::new(); + for metadata_location in metadata_locations { + if !is_valid_table_metadata_location(namespace, table, metadata_location) { + continue; + } + let Some(metadata_object) = backend.read_object(table_bucket, metadata_location).await? else { + continue; + }; + let Ok(metadata) = serde_json::from_slice::(&metadata_object.data) else { + continue; + }; + if metadata_contains_protected_snapshot_ref(&metadata, &protected_snapshot_ids) { + retained.insert(metadata_location.clone()); + } + } + Ok(retained) +} + +pub(crate) fn protected_ref_snapshot_ids(current_metadata: &serde_json::Value) -> BTreeSet { + let mut snapshot_ids = BTreeSet::new(); + let current_snapshot_id = current_metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64); + let Some(refs) = current_metadata.get("refs").and_then(serde_json::Value::as_object) else { + return snapshot_ids; + }; + + for reference in refs.values() { + if let Some(snapshot_id) = reference.get("snapshot-id").and_then(serde_json::Value::as_i64) + && Some(snapshot_id) != current_snapshot_id + { + snapshot_ids.insert(snapshot_id); + } + } + snapshot_ids +} + +pub(crate) fn metadata_contains_protected_snapshot_ref( + metadata: &serde_json::Value, + protected_snapshot_ids: &BTreeSet, +) -> bool { + let current_snapshot_matches = metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_some_and(|snapshot_id| protected_snapshot_ids.contains(&snapshot_id)); + if current_snapshot_matches { + return true; + } + + let Some(refs) = metadata.get("refs").and_then(serde_json::Value::as_object) else { + return false; + }; + refs.values().any(|reference| { + reference + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_some_and(|snapshot_id| protected_snapshot_ids.contains(&snapshot_id)) + }) +} + +pub(crate) fn metadata_candidate_is_past_safety_window(mod_time: Option, now: OffsetDateTime) -> bool { + let Some(mod_time) = mod_time else { + return false; + }; + mod_time <= now - Duration::seconds(TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS) +} diff --git a/rustfs/src/table_catalog/iceberg/mod.rs b/rustfs/src/table_catalog/iceberg/mod.rs new file mode 100644 index 000000000..99cd303f6 --- /dev/null +++ b/rustfs/src/table_catalog/iceberg/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod commit; +mod manifest; +mod metadata; +mod validation; + +pub(crate) use commit::*; +pub(crate) use manifest::*; +pub(crate) use metadata::*; +pub(crate) use validation::*; diff --git a/rustfs/src/table_catalog/iceberg/validation.rs b/rustfs/src/table_catalog/iceberg/validation.rs new file mode 100644 index 000000000..24ab10003 --- /dev/null +++ b/rustfs/src/table_catalog/iceberg/validation.rs @@ -0,0 +1,219 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::*; + +fn normalize_warehouse_object_prefix(object_prefix: &str, max_prefix_depth: Option) -> TableCatalogStoreResult { + let object_prefix = object_prefix.strip_suffix('/').unwrap_or(object_prefix); + if object_prefix.is_empty() { + return Err(TableCatalogStoreError::Invalid( + "table warehouse location must include an object prefix".to_string(), + )); + } + if object_prefix.contains('\\') { + return Err(TableCatalogStoreError::Invalid( + "table warehouse location contains an invalid path separator".to_string(), + )); + } + let mut segment_count = 0; + for segment in object_prefix.split('/') { + segment_count += 1; + if segment.is_empty() || segment == "." || segment == ".." { + return Err(TableCatalogStoreError::Invalid( + "table warehouse location contains an invalid path segment".to_string(), + )); + } + } + if max_prefix_depth.is_some_and(|max_prefix_depth| segment_count > max_prefix_depth) { + return Err(TableCatalogStoreError::Invalid( + "table warehouse location exceeds the maximum prefix depth".to_string(), + )); + } + + let mut normalized = object_prefix.to_string(); + normalized.push('/'); + Ok(normalized) +} + +fn warehouse_object_prefix_from_location( + table_bucket: &str, + warehouse_location: &str, + max_prefix_depth: Option, +) -> TableCatalogStoreResult { + let location = warehouse_location + .strip_prefix("s3://") + .ok_or_else(|| TableCatalogStoreError::Invalid("table warehouse location must be an s3 URI".to_string()))?; + let (bucket, object_prefix) = location + .split_once('/') + .ok_or_else(|| TableCatalogStoreError::Invalid("table warehouse location must include an object prefix".to_string()))?; + if bucket != table_bucket { + return Err(TableCatalogStoreError::Invalid( + "table warehouse location must be inside the table bucket".to_string(), + )); + } + normalize_warehouse_object_prefix(object_prefix, max_prefix_depth) +} + +fn table_warehouse_object_prefix_from_location(table_bucket: &str, warehouse_location: &str) -> TableCatalogStoreResult { + warehouse_object_prefix_from_location(table_bucket, warehouse_location, Some(WAREHOUSE_INDEX_MAX_PREFIX_DEPTH)) +} + +fn view_warehouse_object_prefix_from_location(table_bucket: &str, warehouse_location: &str) -> TableCatalogStoreResult { + warehouse_object_prefix_from_location(table_bucket, warehouse_location, None) +} + +pub(crate) fn validate_table_warehouse_location(table_bucket: &str, warehouse_location: &str) -> TableCatalogStoreResult<()> { + table_warehouse_object_prefix_from_location(table_bucket, warehouse_location).map(|_| ()) +} + +pub(crate) fn validate_view_warehouse_location(table_bucket: &str, warehouse_location: &str) -> TableCatalogStoreResult<()> { + view_warehouse_object_prefix_from_location(table_bucket, warehouse_location).map(|_| ()) +} + +pub(crate) fn table_warehouse_object_prefix(entry: &TableEntry) -> TableCatalogStoreResult { + table_warehouse_object_prefix_from_location(&entry.table_bucket, &entry.warehouse_location) +} + +pub(crate) fn table_warehouse_index_entry(entry: &TableEntry) -> TableCatalogStoreResult { + Ok(TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: entry.table_bucket.clone(), + namespace: entry.namespace.clone(), + table: entry.table.clone(), + table_id: entry.table_id.clone(), + warehouse_object_prefix: table_warehouse_object_prefix(entry)?, + state: entry.state.clone(), + }) +} + +fn table_warehouse_data_dir_path(entry: &TableEntry) -> TableCatalogStoreResult { + Ok(format!("{}{}", table_warehouse_object_prefix(entry)?, DATA_DIR)) +} + +pub(crate) fn table_object_s3_location(table_bucket: &str, object_key: &str) -> String { + format!("s3://{table_bucket}/{object_key}") +} + +fn metadata_warehouse_location( + table_bucket: &str, + metadata_location: &str, + metadata_object: &TableCatalogObject, + validate_location: fn(&str, &str) -> TableCatalogStoreResult<()>, +) -> TableCatalogStoreResult> { + let metadata: serde_json::Value = serde_json::from_slice(&metadata_object.data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to parse new metadata {metadata_location}: {err}")))?; + let Some(location) = metadata.get("location").and_then(serde_json::Value::as_str) else { + return Ok(None); + }; + validate_location(table_bucket, location)?; + Ok(Some(location.to_string())) +} + +pub(crate) fn table_metadata_warehouse_location( + table_bucket: &str, + metadata_location: &str, + metadata_object: &TableCatalogObject, +) -> TableCatalogStoreResult> { + metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_table_warehouse_location) +} + +pub(crate) fn view_metadata_warehouse_location( + table_bucket: &str, + metadata_location: &str, + metadata_object: &TableCatalogObject, +) -> TableCatalogStoreResult> { + metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_view_warehouse_location) +} + +pub(crate) fn warehouse_index_candidate_prefixes(object: &str) -> Vec<&str> { + let mut prefixes = Vec::new(); + for (index, byte) in object.as_bytes().iter().enumerate() { + if *byte == b'/' { + prefixes.push(&object[..=index]); + if prefixes.len() >= WAREHOUSE_INDEX_MAX_PREFIX_DEPTH { + break; + } + } + } + prefixes.reverse(); + prefixes +} + +pub(crate) fn table_data_plane_resource_from_entry(table: TableEntry, warehouse_object_prefix: String) -> TableDataPlaneResource { + TableDataPlaneResource { + table_bucket: table.table_bucket, + namespace: table.namespace, + table: table.table, + table_id: table.table_id, + warehouse_object_prefix, + } +} + +pub(crate) async fn table_data_plane_resource_for_object( + store: &S, + bucket: &str, + object: &str, +) -> TableCatalogStoreResult> +where + S: TableCatalogStore + ?Sized, +{ + store.resolve_table_data_plane_resource(bucket, object).await +} + +pub(crate) async fn scan_table_data_plane_resource_for_object( + store: &S, + bucket: &str, + object: &str, +) -> TableCatalogStoreResult> +where + S: TableCatalogStore + ?Sized, +{ + if bucket.is_empty() || object.is_empty() { + return Ok(None); + } + + let Some(table_bucket) = store.get_table_bucket(bucket).await? else { + return Ok(None); + }; + if table_bucket.state != TableCatalogEntryState::Active { + return Ok(None); + } + + let mut matched: Option = None; + for namespace in store.list_namespaces(bucket).await? { + if namespace.state != TableCatalogEntryState::Active { + continue; + } + for table in store.list_tables(bucket, &namespace.namespace).await? { + if table.state != TableCatalogEntryState::Active { + continue; + } + let Ok(warehouse_object_prefix) = table_warehouse_object_prefix(&table) else { + continue; + }; + if !object.starts_with(&warehouse_object_prefix) { + continue; + } + if matched + .as_ref() + .is_some_and(|current| current.warehouse_object_prefix.len() >= warehouse_object_prefix.len()) + { + continue; + } + matched = Some(table_data_plane_resource_from_entry(table, warehouse_object_prefix)); + } + } + + Ok(matched) +} diff --git a/rustfs/src/table_catalog/identifier.rs b/rustfs/src/table_catalog/identifier.rs new file mode 100644 index 000000000..b4576a72a --- /dev/null +++ b/rustfs/src/table_catalog/identifier.rs @@ -0,0 +1,355 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IdentifierSegment(String); + +impl IdentifierSegment { + pub const MAX_LEN: usize = 64; + + pub fn parse(value: impl Into) -> Result { + let value = value.into(); + validate_identifier_segment(&value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Namespace { + segments: Vec, +} + +impl Namespace { + pub const MAX_LEN: usize = 512; + + pub fn parse(value: &str) -> Result { + if value.is_empty() { + return Err(CatalogIdentifierError::Empty); + } + if value.len() > Self::MAX_LEN { + return Err(CatalogIdentifierError::NamespaceTooLong { max: Self::MAX_LEN }); + } + + let mut segments = Vec::new(); + for segment in value.split('.') { + segments.push(IdentifierSegment::parse(segment.to_string())?); + } + + Ok(Self { segments }) + } + + pub fn segments(&self) -> &[IdentifierSegment] { + &self.segments + } + + pub fn storage_id(&self) -> String { + self.segments + .iter() + .map(IdentifierSegment::as_str) + .collect::>() + .join("/") + } + + pub fn public_name(&self) -> String { + self.segments + .iter() + .map(IdentifierSegment::as_str) + .collect::>() + .join(".") + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableIdentifier { + warehouse: IdentifierSegment, + namespace: Namespace, + name: IdentifierSegment, +} + +impl TableIdentifier { + pub fn new(warehouse: IdentifierSegment, namespace: Namespace, name: IdentifierSegment) -> Self { + Self { + warehouse, + namespace, + name, + } + } + + pub fn warehouse(&self) -> &IdentifierSegment { + &self.warehouse + } + + pub fn namespace(&self) -> &Namespace { + &self.namespace + } + + pub fn name(&self) -> &IdentifierSegment { + &self.name + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TablePathResolver { + reserved_prefix: &'static str, +} + +impl Default for TablePathResolver { + fn default() -> Self { + Self { + reserved_prefix: TABLE_RESERVED_PREFIX, + } + } +} + +impl TablePathResolver { + pub fn current_pointer_path(&self, table: &TableIdentifier) -> String { + format!("{}/{}", self.table_root(table), CURRENT_POINTER_FILE) + } + + pub fn metadata_dir_path(&self, table: &TableIdentifier) -> String { + format!("{}/{}", self.table_root(table), METADATA_DIR) + } + + pub fn metadata_file_path(&self, table: &TableIdentifier, metadata_file_name: &str) -> String { + format!("{}/{}", self.metadata_dir_path(table), metadata_file_name) + } + + fn table_root(&self, table: &TableIdentifier) -> String { + format!( + "{}/{}/{}/{}/{}/{}/{}", + self.reserved_prefix, + WAREHOUSE_ROOT, + table.warehouse().as_str(), + NAMESPACE_ROOT, + table.namespace().storage_id(), + TABLE_ROOT, + table.name().as_str() + ) + } +} + +pub fn is_reserved_table_object_key(object_key: &str) -> bool { + object_key == TABLE_RESERVED_PREFIX + || object_key + .strip_prefix(TABLE_RESERVED_PREFIX) + .is_some_and(|rest| rest.starts_with('/')) +} + +pub(crate) fn default_namespace_root_prefix() -> String { + format!( + "{}/{}/{}/{}/", + TABLE_RESERVED_PREFIX, WAREHOUSE_ROOT, DEFAULT_WAREHOUSE_ID, NAMESPACE_ROOT + ) +} + +pub(crate) fn default_namespace_marker_path(namespace: &Namespace) -> String { + format!("{}{}/{}", default_namespace_root_prefix(), namespace.storage_id(), NAMESPACE_MARKER_FILE) +} + +pub(crate) fn default_table_root_prefix(namespace: &Namespace) -> String { + format!("{}{}/{}/", default_namespace_root_prefix(), namespace.storage_id(), TABLE_ROOT) +} + +pub(crate) fn default_table_marker_path(namespace: &Namespace, table: &IdentifierSegment) -> String { + format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), TABLE_MARKER_FILE) +} + +pub(crate) fn default_table_metadata_dir_path(namespace: &Namespace, table: &IdentifierSegment) -> String { + format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), METADATA_DIR) +} + +pub(crate) fn default_table_data_dir_path(namespace: &Namespace, table: &IdentifierSegment) -> String { + format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), DATA_DIR) +} + +pub(crate) fn default_table_delete_dir_path(namespace: &Namespace, table: &IdentifierSegment) -> String { + format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), DELETE_DIR) +} + +pub(crate) fn default_view_root_prefix(namespace: &Namespace) -> String { + format!("{}{}/{}/", default_namespace_root_prefix(), namespace.storage_id(), VIEW_ROOT) +} + +pub(crate) fn default_view_metadata_dir_path(namespace: &Namespace, view: &IdentifierSegment) -> String { + format!("{}{}/{}", default_view_root_prefix(namespace), view.as_str(), METADATA_DIR) +} + +pub(crate) fn default_view_metadata_file_path( + namespace: &Namespace, + view: &IdentifierSegment, + metadata_file_name: &str, +) -> String { + format!("{}/{}", default_view_metadata_dir_path(namespace, view), metadata_file_name) +} + +pub(crate) fn default_table_metadata_file_path( + namespace: &Namespace, + table: &IdentifierSegment, + metadata_file_name: &str, +) -> String { + format!("{}/{}", default_table_metadata_dir_path(namespace, table), metadata_file_name) +} + +pub(crate) fn default_table_current_pointer_path(namespace: &Namespace, table: &IdentifierSegment) -> String { + format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), CURRENT_POINTER_FILE) +} + +pub(crate) fn default_table_lifecycle_path(namespace: &Namespace, table: &IdentifierSegment) -> String { + format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), LIFECYCLE_FILE) +} + +pub(crate) fn namespace_name_from_marker_path(object_key: &str) -> Option { + let prefix = default_namespace_root_prefix(); + let suffix = format!("/{NAMESPACE_MARKER_FILE}"); + + object_key + .strip_prefix(prefix.as_str()) + .and_then(|value| value.strip_suffix(suffix.as_str())) + .filter(|value| !value.is_empty()) + .map(|value| value.replace('/', ".")) +} + +pub(crate) fn table_name_from_marker_path(namespace: &Namespace, object_key: &str) -> Option { + let prefix = default_table_root_prefix(namespace); + let suffix = format!("/{TABLE_MARKER_FILE}"); + + object_key + .strip_prefix(prefix.as_str()) + .and_then(|value| value.strip_suffix(suffix.as_str())) + .filter(|value| !value.is_empty() && !value.contains('/')) + .map(ToString::to_string) +} + +pub(crate) fn metadata_location_from_metadata_file_path( + namespace: &Namespace, + table: &IdentifierSegment, + object_key: &str, +) -> Option { + let prefix = format!("{}/", default_table_metadata_dir_path(namespace, table)); + + object_key + .strip_prefix(prefix.as_str()) + .filter(|value| is_valid_table_metadata_file_name(value)) + .map(|_| object_key.to_string()) +} + +pub(crate) fn is_valid_table_metadata_location( + namespace: &Namespace, + table: &IdentifierSegment, + metadata_location: &str, +) -> bool { + if metadata_location.is_empty() { + return false; + } + + let metadata_prefix = format!("{}/", default_table_metadata_dir_path(namespace, table)); + metadata_location + .strip_prefix(&metadata_prefix) + .is_some_and(is_valid_table_metadata_file_name) +} + +pub(crate) fn is_valid_view_metadata_location(namespace: &Namespace, view: &IdentifierSegment, metadata_location: &str) -> bool { + if metadata_location.is_empty() { + return false; + } + + let metadata_prefix = format!("{}/", default_view_metadata_dir_path(namespace, view)); + metadata_location + .strip_prefix(&metadata_prefix) + .is_some_and(is_valid_table_metadata_file_name) +} + +pub(crate) fn is_valid_table_metadata_file_name(metadata_file_name: &str) -> bool { + if metadata_file_name.is_empty() + || metadata_file_name.len() > TABLE_METADATA_FILE_NAME_MAX_LEN + || !metadata_file_name.ends_with(".json") + || metadata_file_name.contains("..") + || metadata_file_name.contains('%') + || metadata_file_name.contains('/') + || metadata_file_name.contains('\\') + || metadata_file_name.bytes().any(|byte| byte.is_ascii_control()) + { + return false; + } + + let bytes = metadata_file_name.as_bytes(); + if !is_lower_ascii_alnum(bytes[0]) || !is_lower_ascii_alnum(bytes[bytes.len() - 1]) { + return false; + } + + bytes + .iter() + .all(|byte| is_lower_ascii_alnum(*byte) || matches!(*byte, b'.' | b'_' | b'-')) +} + +pub fn validate_object_mutation(table_bucket_enabled: bool, object_key: &str) -> Result<(), TableObjectMutationError> { + if table_bucket_enabled && is_reserved_table_object_key(object_key) { + return Err(TableObjectMutationError::ReservedCatalogObject); + } + + Ok(()) +} + +pub(crate) async fn validate_bucket_object_mutation(bucket: &str, object_key: &str) -> Result<(), TableObjectMutationError> { + if !is_reserved_table_object_key(object_key) { + return Ok(()); + } + + let table_bucket_enabled = get_bucket_metadata(bucket) + .await + .map(|metadata| metadata.table_bucket_enabled()) + .unwrap_or(true); + + validate_object_mutation(table_bucket_enabled, object_key) +} + +fn validate_identifier_segment(value: &str) -> Result<(), CatalogIdentifierError> { + if value.is_empty() { + return Err(CatalogIdentifierError::Empty); + } + + if value.len() > IdentifierSegment::MAX_LEN { + return Err(CatalogIdentifierError::TooLong { + max: IdentifierSegment::MAX_LEN, + }); + } + + if matches!(value, "." | "..") || value.contains('%') || value.contains('/') || value.contains('\\') { + return Err(CatalogIdentifierError::Ambiguous); + } + + let bytes = value.as_bytes(); + if !is_lower_ascii_alnum(bytes[0]) || !is_lower_ascii_alnum(bytes[bytes.len() - 1]) { + return Err(CatalogIdentifierError::InvalidBoundary); + } + + if bytes + .iter() + .any(|byte| !is_lower_ascii_alnum(*byte) && !matches!(*byte, b'_' | b'-')) + { + return Err(CatalogIdentifierError::InvalidCharacter); + } + + Ok(()) +} + +fn is_lower_ascii_alnum(value: u8) -> bool { + value.is_ascii_lowercase() || value.is_ascii_digit() +} diff --git a/rustfs/src/table_catalog/maintenance/config.rs b/rustfs/src/table_catalog/maintenance/config.rs new file mode 100644 index 000000000..469a4d697 --- /dev/null +++ b/rustfs/src/table_catalog/maintenance/config.rs @@ -0,0 +1,113 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::*; + +pub(crate) fn validate_catalog_entry_version(kind: &str, version: u16) -> TableCatalogStoreResult<()> { + if version != TABLE_CATALOG_ENTRY_VERSION { + return Err(TableCatalogStoreError::Invalid(format!("unsupported {kind} entry version"))); + } + Ok(()) +} + +fn validate_table_maintenance_config_version(version: u16) -> TableCatalogStoreResult<()> { + if version != TABLE_MAINTENANCE_CONFIG_VERSION { + return Err(TableCatalogStoreError::Invalid( + "unsupported table maintenance config entry version".to_string(), + )); + } + Ok(()) +} + +pub(crate) fn validate_table_maintenance_config(config: &TableMaintenanceConfig) -> TableCatalogStoreResult<()> { + validate_table_maintenance_config_version(config.version)?; + if config.worker_lease_timeout_seconds == 0 { + return Err(TableCatalogStoreError::Invalid( + "worker-lease-timeout-seconds must be greater than zero".to_string(), + )); + } + if config.worker_lease_timeout_seconds > TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_MAX_SECONDS { + return Err(TableCatalogStoreError::Invalid(format!( + "worker-lease-timeout-seconds cannot exceed {TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_MAX_SECONDS}" + ))); + } + if config.max_retry_attempts > 10 { + return Err(TableCatalogStoreError::Invalid("max-retry-attempts cannot exceed 10".to_string())); + } + if config.max_retry_attempts > 0 && config.retry_initial_backoff_seconds == 0 { + return Err(TableCatalogStoreError::Invalid( + "retry-initial-backoff-seconds must be greater than zero when retry is enabled".to_string(), + )); + } + if config.max_retry_attempts > 0 && config.retry_initial_backoff_seconds > TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS { + return Err(TableCatalogStoreError::Invalid(format!( + "retry-initial-backoff-seconds cannot exceed {TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS}" + ))); + } + if config.max_retry_attempts > 0 && config.retry_max_backoff_seconds > TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS { + return Err(TableCatalogStoreError::Invalid(format!( + "retry-max-backoff-seconds cannot exceed {TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS}" + ))); + } + if config.max_retry_attempts > 0 && config.retry_max_backoff_seconds < config.retry_initial_backoff_seconds { + return Err(TableCatalogStoreError::Invalid( + "retry-max-backoff-seconds must be greater than or equal to retry-initial-backoff-seconds".to_string(), + )); + } + if config.quarantine_enabled && config.quarantine_retention_seconds == 0 { + return Err(TableCatalogStoreError::Invalid( + "quarantine-retention-seconds must be greater than zero when quarantine is enabled".to_string(), + )); + } + Ok(()) +} + +pub(crate) fn validate_table_snapshot_expiration_config(config: &TableSnapshotExpirationConfig) -> TableCatalogStoreResult<()> { + if config.min_snapshots_to_keep == 0 { + return Err(TableCatalogStoreError::Invalid( + "min-snapshots-to-keep must be greater than zero".to_string(), + )); + } + if config.max_snapshot_age_ms < 0 { + return Err(TableCatalogStoreError::Invalid("max-snapshot-age-ms cannot be negative".to_string())); + } + Ok(()) +} + +pub(crate) fn validate_table_compaction_planning_config(config: &TableCompactionPlanningConfig) -> TableCatalogStoreResult<()> { + if config.target_file_size_bytes == 0 { + return Err(TableCatalogStoreError::Invalid( + "target-file-size-bytes must be greater than zero".to_string(), + )); + } + if config.small_file_threshold_bytes == 0 { + return Err(TableCatalogStoreError::Invalid( + "small-file-threshold-bytes must be greater than zero".to_string(), + )); + } + if config.small_file_threshold_bytes > config.target_file_size_bytes { + return Err(TableCatalogStoreError::Invalid( + "small-file-threshold-bytes cannot exceed target-file-size-bytes".to_string(), + )); + } + if config.min_input_files < 2 { + return Err(TableCatalogStoreError::Invalid("min-input-files must be at least two".to_string())); + } + if config.max_rewrite_bytes_per_job < config.target_file_size_bytes { + return Err(TableCatalogStoreError::Invalid( + "max-rewrite-bytes-per-job must be at least target-file-size-bytes".to_string(), + )); + } + Ok(()) +} diff --git a/rustfs/src/table_catalog/maintenance/mod.rs b/rustfs/src/table_catalog/maintenance/mod.rs new file mode 100644 index 000000000..e38b5a4de --- /dev/null +++ b/rustfs/src/table_catalog/maintenance/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod config; +mod planner; +mod recovery; +mod worker; + +pub(crate) use config::*; +pub(crate) use planner::*; +pub(crate) use recovery::*; +pub(crate) use worker::*; diff --git a/rustfs/src/table_catalog/maintenance/planner.rs b/rustfs/src/table_catalog/maintenance/planner.rs new file mode 100644 index 000000000..75f31be73 --- /dev/null +++ b/rustfs/src/table_catalog/maintenance/planner.rs @@ -0,0 +1,1421 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::*; + +#[derive(Debug)] +struct TableSnapshotExpirationDraft { + snapshot_id: Option, + sequence_number: Option, + timestamp_ms: Option, + manifest_list: Option, + reasons: BTreeSet, +} + +pub(crate) fn table_snapshot_expiration_report( + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + entry: &TableEntry, + current_metadata: &serde_json::Value, + config: TableSnapshotExpirationConfig, + now: OffsetDateTime, +) -> TableSnapshotExpirationReport { + let current_snapshot_id = current_metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64); + let expiration_watermark_ms = unix_timestamp_millis(now).saturating_sub(config.max_snapshot_age_ms); + let (protected_ref_snapshot_ids, user_defined_ref_snapshot_ids, ref_retention_conflict_snapshot_ids) = + snapshot_expiration_ref_state(current_metadata, current_snapshot_id); + let table_retention_property_conflict = snapshot_expiration_table_property_conflicts(current_metadata, &config); + + let mut drafts = snapshot_expiration_drafts(current_metadata, current_snapshot_id); + mark_recent_snapshots_to_keep(&mut drafts, config.min_snapshots_to_keep); + + let mut snapshot_reports = Vec::with_capacity(drafts.len()); + for mut draft in drafts { + if let Some(snapshot_id) = draft.snapshot_id { + if protected_ref_snapshot_ids.contains(&snapshot_id) { + draft.reasons.insert(TableSnapshotExpirationReason::ProtectedSnapshotRef); + } + if user_defined_ref_snapshot_ids.contains(&snapshot_id) { + draft.reasons.insert(TableSnapshotExpirationReason::UserDefinedSnapshotRef); + } + if ref_retention_conflict_snapshot_ids.contains(&snapshot_id) { + draft + .reasons + .insert(TableSnapshotExpirationReason::SnapshotRefRetentionConflict); + } + } + if table_retention_property_conflict { + draft + .reasons + .insert(TableSnapshotExpirationReason::TableRetentionPropertyConflict); + } + + let state = if snapshot_expiration_requires_manual_review(&draft.reasons) { + TableSnapshotExpirationSnapshotState::ManualReviewRequired + } else if snapshot_expiration_is_retained(&draft.reasons) { + TableSnapshotExpirationSnapshotState::Retained + } else if let Some(timestamp_ms) = draft.timestamp_ms { + if timestamp_ms <= expiration_watermark_ms { + draft.reasons.insert(TableSnapshotExpirationReason::SnapshotAgeExpired); + TableSnapshotExpirationSnapshotState::ExpirationCandidate + } else { + draft + .reasons + .insert(TableSnapshotExpirationReason::SnapshotAgeWithinRetention); + TableSnapshotExpirationSnapshotState::Retained + } + } else { + draft.reasons.insert(TableSnapshotExpirationReason::MissingSnapshotTimestamp); + TableSnapshotExpirationSnapshotState::ManualReviewRequired + }; + + snapshot_reports.push(TableSnapshotExpirationSnapshotReport { + snapshot_id: draft.snapshot_id, + sequence_number: draft.sequence_number, + timestamp_ms: draft.timestamp_ms, + manifest_list: draft.manifest_list, + state, + reasons: draft.reasons.into_iter().collect(), + }); + } + + let retained_snapshot_count = snapshot_reports + .iter() + .filter(|snapshot| snapshot.state == TableSnapshotExpirationSnapshotState::Retained) + .count(); + let expiration_candidate_count = snapshot_reports + .iter() + .filter(|snapshot| snapshot.state == TableSnapshotExpirationSnapshotState::ExpirationCandidate) + .count(); + let manual_review_count = snapshot_reports + .iter() + .filter(|snapshot| snapshot.state == TableSnapshotExpirationSnapshotState::ManualReviewRequired) + .count(); + + TableSnapshotExpirationReport { + table_bucket: table_bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: entry.table_id.clone(), + current_metadata_location: entry.metadata_location.clone(), + current_snapshot_id, + config, + expiration_watermark_ms, + retained_snapshot_count, + expiration_candidate_count, + manual_review_count, + expired_snapshot_ids: Vec::new(), + committed_metadata_location: None, + snapshot_reports, + } +} + +#[derive(Debug, Clone)] +struct TableCompactionDataFileCandidate { + location: String, + size_bytes: u64, + rewrite_prefix: String, + sort_order_id: Option, +} + +#[derive(Debug, Default)] +struct CompactionManifestPlanning { + candidates: Vec, + row_level_planning: TableRowLevelMaintenancePlanningReport, +} + +pub(crate) struct CompactedParquetFile { + pub(crate) data: Vec, + pub(crate) record_count: u64, +} + +#[derive(Debug, Clone)] +pub(crate) struct CompactedDataFile { + pub(crate) object_key: String, + pub(crate) file_path: String, + pub(crate) file_size_bytes: u64, + pub(crate) record_count: u64, + pub(crate) partition_spec_id: i32, + pub(crate) partition: Vec<(String, apache_avro::types::Value)>, + pub(crate) sort_order_id: Option, + pub(crate) status: i32, + pub(crate) snapshot_id: i64, + pub(crate) sequence_number: i64, + pub(crate) file_sequence_number: i64, +} + +pub(crate) struct CompactionManifestListSummary<'a> { + pub(crate) manifest_path: &'a str, + pub(crate) manifest_length: u64, + pub(crate) partition_spec_id: i32, + pub(crate) snapshot_id: i64, + pub(crate) sequence_number: i64, + pub(crate) added_files_count: usize, + pub(crate) existing_files_count: usize, + pub(crate) added_rows_count: u64, + pub(crate) existing_rows_count: u64, +} + +pub(crate) async fn table_compaction_planning_report( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + entry: &TableEntry, + current_metadata: &serde_json::Value, + config: TableCompactionPlanningConfig, +) -> TableCatalogStoreResult +where + B: TableCatalogObjectBackend, +{ + let current_snapshot_id = current_metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64); + let warehouse_object_prefix = table_warehouse_object_prefix(entry).ok(); + let mut snapshot_reports = Vec::new(); + let mut candidates = Vec::new(); + let mut rewrite_groups = Vec::new(); + let mut row_level_planning = TableRowLevelMaintenancePlanningReport::default(); + + if let Some(current_snapshot_id) = current_snapshot_id { + let current_snapshot = current_metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .find(|snapshot| { + snapshot + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_some_and(|snapshot_id| snapshot_id == current_snapshot_id) + }); + match current_snapshot { + Some(snapshot) => { + let manifest_list = snapshot + .get("manifest-list") + .and_then(serde_json::Value::as_str) + .map(ToString::to_string); + match manifest_list.as_deref() { + Some(manifest_list) => { + let planning = match compaction_data_file_candidates( + backend, + table_bucket, + namespace, + table, + warehouse_object_prefix.as_deref(), + manifest_list, + &config, + ) + .await + { + Ok(planning) => planning, + Err(_) => { + snapshot_reports.push(TableCompactionSnapshotReport { + snapshot_id: Some(current_snapshot_id), + manifest_list: Some(manifest_list.to_string()), + status: TableCompactionPlanningStatus::ManualReviewRequired, + reasons: vec![ + TableCompactionPlanningReason::ManifestList, + TableCompactionPlanningReason::ManifestAvroReaderUnavailable, + ], + }); + CompactionManifestPlanning::default() + } + }; + row_level_planning = planning.row_level_planning; + candidates = planning.candidates; + if row_level_planning.status == TableRowLevelMaintenancePlanningStatus::ManualReviewRequired + && snapshot_reports.is_empty() + { + snapshot_reports.push(TableCompactionSnapshotReport { + snapshot_id: Some(current_snapshot_id), + manifest_list: Some(manifest_list.to_string()), + status: TableCompactionPlanningStatus::ManualReviewRequired, + reasons: compaction_row_level_planning_reasons(&row_level_planning), + }); + } else if !candidates.is_empty() && snapshot_reports.is_empty() { + rewrite_groups = compaction_rewrite_groups(&candidates, &config); + let (status, reasons) = if rewrite_groups.is_empty() { + ( + TableCompactionPlanningStatus::NoCandidates, + vec![ + TableCompactionPlanningReason::ManifestList, + TableCompactionPlanningReason::ManifestFile, + ], + ) + } else { + ( + TableCompactionPlanningStatus::RewriteCandidates, + vec![ + TableCompactionPlanningReason::ManifestList, + TableCompactionPlanningReason::ManifestFile, + TableCompactionPlanningReason::SmallDataFile, + TableCompactionPlanningReason::RewriteGroup, + ], + ) + }; + snapshot_reports.push(TableCompactionSnapshotReport { + snapshot_id: Some(current_snapshot_id), + manifest_list: Some(manifest_list.to_string()), + status, + reasons, + }); + } + } + None => snapshot_reports.push(TableCompactionSnapshotReport { + snapshot_id: Some(current_snapshot_id), + manifest_list: None, + status: TableCompactionPlanningStatus::ManualReviewRequired, + reasons: vec![TableCompactionPlanningReason::MissingManifestList], + }), + } + } + None => snapshot_reports.push(TableCompactionSnapshotReport { + snapshot_id: Some(current_snapshot_id), + manifest_list: None, + status: TableCompactionPlanningStatus::ManualReviewRequired, + reasons: vec![TableCompactionPlanningReason::MissingCurrentSnapshot], + }), + } + } + + let manual_review_count = snapshot_reports + .iter() + .filter(|snapshot| snapshot.status == TableCompactionPlanningStatus::ManualReviewRequired) + .count(); + let status = if manual_review_count > 0 { + TableCompactionPlanningStatus::ManualReviewRequired + } else if rewrite_groups.is_empty() { + TableCompactionPlanningStatus::NoCandidates + } else { + TableCompactionPlanningStatus::RewriteCandidates + }; + + Ok(TableCompactionPlanningReport { + table_bucket: table_bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: entry.table_id.clone(), + current_metadata_location: entry.metadata_location.clone(), + current_snapshot_id, + config, + status, + candidate_file_count: candidates.len(), + rewrite_group_count: rewrite_groups.len(), + manual_review_count, + committed_metadata_location: None, + row_level_planning, + rewrite_groups, + snapshot_reports, + }) +} + +async fn compaction_data_file_candidates( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + manifest_list: &str, + config: &TableCompactionPlanningConfig, +) -> TableCatalogStoreResult +where + B: TableCatalogObjectBackend, +{ + let Some(manifest_list_key) = table_catalog_object_key_from_location(table_bucket, manifest_list) else { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest list must be inside the table bucket".to_string(), + )); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &manifest_list_key) + != Some(TableMetadataMaintenanceObjectKind::ManifestList) + { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest list must be inside the table metadata directory".to_string(), + )); + } + let Some(manifest_list_object) = backend.read_object(table_bucket, &manifest_list_key).await? else { + return Err(TableCatalogStoreError::NotFound(format!("compaction manifest list {manifest_list_key}"))); + }; + let manifest_paths = manifest_paths_from_manifest_list_avro(&manifest_list_object.data)?; + let mut planning = CompactionManifestPlanning::default(); + for manifest_location in manifest_paths { + let Some(manifest_key) = table_catalog_object_key_from_location(table_bucket, &manifest_location) else { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest must be inside the table bucket".to_string(), + )); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &manifest_key) + != Some(TableMetadataMaintenanceObjectKind::ManifestFile) + { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest must be inside the table metadata directory".to_string(), + )); + } + let Some(manifest_object) = backend.read_object(table_bucket, &manifest_key).await? else { + return Err(TableCatalogStoreError::NotFound(format!("compaction manifest {manifest_key}"))); + }; + for reference in data_file_references_from_manifest_avro(&manifest_object.data)? { + if reference.object_kind != TableMetadataMaintenanceObjectKind::DataFile { + record_compaction_row_level_delete_file( + backend, + table_bucket, + namespace, + table, + warehouse_object_prefix, + &mut planning.row_level_planning, + &reference, + ) + .await?; + continue; + } + validate_compaction_manifest_entry_status(reference.entry_status)?; + let Some(data_key) = table_catalog_object_key_from_location(table_bucket, &reference.location) else { + return Err(TableCatalogStoreError::Invalid( + "compaction data file must be inside the table bucket".to_string(), + )); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &data_key) + != Some(TableMetadataMaintenanceObjectKind::DataFile) + { + return Err(TableCatalogStoreError::Invalid( + "compaction data file must be inside the table data directory".to_string(), + )); + } + let Some(data_object) = backend.read_object(table_bucket, &data_key).await? else { + return Err(TableCatalogStoreError::NotFound(format!("compaction data file {data_key}"))); + }; + let size_bytes = u64::try_from(data_object.data.len()).unwrap_or(u64::MAX); + if size_bytes <= config.small_file_threshold_bytes { + planning.candidates.push(TableCompactionDataFileCandidate { + rewrite_prefix: compaction_data_file_rewrite_prefix(namespace, table, warehouse_object_prefix, &data_key) + .unwrap_or_else(|| data_key.clone()), + location: data_key, + size_bytes, + sort_order_id: reference.sort_order_id, + }); + } + } + } + Ok(planning) +} + +async fn record_compaction_row_level_delete_file( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + planning: &mut TableRowLevelMaintenancePlanningReport, + reference: &ManifestDataFileReference, +) -> TableCatalogStoreResult<()> +where + B: TableCatalogObjectBackend, +{ + let Some(content) = row_level_delete_file_content(reference.content) else { + return Ok(()); + }; + let Some(delete_key) = table_catalog_object_key_from_location(table_bucket, &reference.location) else { + return Err(TableCatalogStoreError::Invalid( + "compaction delete file must be inside the table bucket".to_string(), + )); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &delete_key) + != Some(TableMetadataMaintenanceObjectKind::DeleteFile) + { + return Err(TableCatalogStoreError::Invalid( + "compaction delete file must be inside the table delete directory".to_string(), + )); + } + + let object_exists = backend.read_object(table_bucket, &delete_key).await?.is_some(); + planning.status = TableRowLevelMaintenancePlanningStatus::ManualReviewRequired; + planning.delete_file_count = planning.delete_file_count.saturating_add(1); + planning.manual_review_count = planning.manual_review_count.saturating_add(1); + push_row_level_planning_reason(planning, TableRowLevelMaintenancePlanningReason::DeleteFileRewriteUnsupported); + match content { + TableRowLevelDeleteFileContent::PositionDelete => { + planning.position_delete_file_count = planning.position_delete_file_count.saturating_add(1); + push_row_level_planning_reason(planning, TableRowLevelMaintenancePlanningReason::PositionDeleteFile); + } + TableRowLevelDeleteFileContent::EqualityDelete => { + planning.equality_delete_file_count = planning.equality_delete_file_count.saturating_add(1); + push_row_level_planning_reason(planning, TableRowLevelMaintenancePlanningReason::EqualityDeleteFile); + } + } + if !object_exists { + push_row_level_planning_reason(planning, TableRowLevelMaintenancePlanningReason::MissingDeleteFile); + } + planning.delete_files.push(TableRowLevelDeleteFilePlanningReport { + file_location: delete_key, + content, + object_exists, + record_count: reference.record_count, + file_size_bytes: reference.file_size_bytes, + sequence_number: reference.sequence_number, + file_sequence_number: reference.file_sequence_number, + }); + Ok(()) +} + +fn row_level_delete_file_content(content: ManifestDataFileContent) -> Option { + match content { + ManifestDataFileContent::Data => None, + ManifestDataFileContent::PositionDelete => Some(TableRowLevelDeleteFileContent::PositionDelete), + ManifestDataFileContent::EqualityDelete => Some(TableRowLevelDeleteFileContent::EqualityDelete), + } +} + +fn push_row_level_planning_reason( + planning: &mut TableRowLevelMaintenancePlanningReport, + reason: TableRowLevelMaintenancePlanningReason, +) { + if !planning.reasons.contains(&reason) { + planning.reasons.push(reason); + } +} + +fn compaction_row_level_planning_reasons( + planning: &TableRowLevelMaintenancePlanningReport, +) -> Vec { + let mut reasons = vec![ + TableCompactionPlanningReason::ManifestList, + TableCompactionPlanningReason::ManifestFile, + TableCompactionPlanningReason::DeleteFile, + TableCompactionPlanningReason::RowLevelRewriteUnsupported, + ]; + if planning.position_delete_file_count > 0 { + reasons.push(TableCompactionPlanningReason::PositionDeleteFile); + } + if planning.equality_delete_file_count > 0 { + reasons.push(TableCompactionPlanningReason::EqualityDeleteFile); + } + reasons +} + +pub(crate) async fn compaction_current_data_files( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + entry: &TableEntry, + current_metadata: &serde_json::Value, +) -> TableCatalogStoreResult> +where + B: TableCatalogObjectBackend, +{ + let current_snapshot_id = current_metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("compaction requires current snapshot metadata".to_string()))?; + let current_snapshot = current_metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .find(|snapshot| { + snapshot + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_some_and(|snapshot_id| snapshot_id == current_snapshot_id) + }) + .ok_or_else(|| TableCatalogStoreError::Invalid("compaction requires current snapshot entry".to_string()))?; + let manifest_list = current_snapshot + .get("manifest-list") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| TableCatalogStoreError::Invalid("compaction requires current snapshot manifest list".to_string()))?; + + let warehouse_object_prefix = table_warehouse_object_prefix(entry).ok(); + let Some(manifest_list_key) = table_catalog_object_key_from_location(table_bucket, manifest_list) else { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest list must be inside the table bucket".to_string(), + )); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix.as_deref(), &manifest_list_key) + != Some(TableMetadataMaintenanceObjectKind::ManifestList) + { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest list must be inside the table metadata directory".to_string(), + )); + } + let Some(manifest_list_object) = backend.read_object(table_bucket, &manifest_list_key).await? else { + return Err(TableCatalogStoreError::NotFound(format!("compaction manifest list {manifest_list_key}"))); + }; + + let mut data_files = Vec::new(); + for manifest_reference in manifest_list_references_from_manifest_list_avro(&manifest_list_object.data)? { + let Some(manifest_key) = table_catalog_object_key_from_location(table_bucket, &manifest_reference.manifest_path) else { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest must be inside the table bucket".to_string(), + )); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix.as_deref(), &manifest_key) + != Some(TableMetadataMaintenanceObjectKind::ManifestFile) + { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest must be inside the table metadata directory".to_string(), + )); + } + let Some(manifest_object) = backend.read_object(table_bucket, &manifest_key).await? else { + return Err(TableCatalogStoreError::NotFound(format!("compaction manifest {manifest_key}"))); + }; + for reference in data_file_references_from_manifest_avro(&manifest_object.data)? { + if reference.object_kind != TableMetadataMaintenanceObjectKind::DataFile { + return Err(TableCatalogStoreError::Invalid( + "compaction currently does not support delete files".to_string(), + )); + } + validate_compaction_manifest_entry_status(reference.entry_status)?; + let Some(data_key) = table_catalog_object_key_from_location(table_bucket, &reference.location) else { + return Err(TableCatalogStoreError::Invalid( + "compaction data file must be inside the table bucket".to_string(), + )); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix.as_deref(), &data_key) + != Some(TableMetadataMaintenanceObjectKind::DataFile) + { + return Err(TableCatalogStoreError::Invalid( + "compaction data file must be inside the table data directory".to_string(), + )); + } + let Some(data_object) = backend.read_object(table_bucket, &data_key).await? else { + return Err(TableCatalogStoreError::NotFound(format!("compaction data file {data_key}"))); + }; + let snapshot_id = reference + .snapshot_id + .or(manifest_reference.added_snapshot_id) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("compaction manifest data file missing snapshot id".to_string()) + })?; + let sequence_number = reference + .sequence_number + .or(manifest_reference.sequence_number) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("compaction manifest data file missing sequence number".to_string()) + })?; + let file_sequence_number = reference + .file_sequence_number + .or(manifest_reference.sequence_number) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("compaction manifest data file missing file sequence number".to_string()) + })?; + data_files.push(CompactedDataFile { + object_key: data_key, + file_path: reference.location, + file_size_bytes: reference + .file_size_bytes + .unwrap_or_else(|| u64::try_from(data_object.data.len()).unwrap_or(u64::MAX)), + record_count: match reference.record_count { + Some(record_count) => record_count, + None => parquet_record_count(&data_object.data)?, + }, + partition_spec_id: manifest_reference.partition_spec_id.unwrap_or(0), + partition: reference.partition, + sort_order_id: reference.sort_order_id, + status: 0, + snapshot_id, + sequence_number, + file_sequence_number, + }); + } + } + + Ok(data_files) +} + +fn compaction_rewrite_groups( + candidates: &[TableCompactionDataFileCandidate], + config: &TableCompactionPlanningConfig, +) -> Vec { + let mut groups = Vec::new(); + let mut candidates_by_prefix = BTreeMap::<(&str, Option), Vec<&TableCompactionDataFileCandidate>>::new(); + for candidate in candidates { + candidates_by_prefix + .entry((candidate.rewrite_prefix.as_str(), candidate.sort_order_id)) + .or_default() + .push(candidate); + } + + for ((_, sort_order_id), prefix_candidates) in candidates_by_prefix { + push_compaction_rewrite_groups_for_prefix(&mut groups, prefix_candidates.as_slice(), sort_order_id, config); + } + groups +} + +fn push_compaction_rewrite_groups_for_prefix( + groups: &mut Vec, + candidates: &[&TableCompactionDataFileCandidate], + sort_order_id: Option, + config: &TableCompactionPlanningConfig, +) { + let mut current_locations = Vec::new(); + let mut current_bytes = 0_u64; + for candidate in candidates { + let next_bytes = current_bytes.saturating_add(candidate.size_bytes); + if !current_locations.is_empty() && next_bytes > config.max_rewrite_bytes_per_job { + push_compaction_rewrite_group(groups, &mut current_locations, &mut current_bytes, sort_order_id, config); + } + current_locations.push(candidate.location.clone()); + current_bytes = current_bytes.saturating_add(candidate.size_bytes); + } + push_compaction_rewrite_group(groups, &mut current_locations, &mut current_bytes, sort_order_id, config); +} + +pub(crate) fn compaction_data_file_rewrite_prefix( + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + location: &str, +) -> Option { + let warehouse_data_prefix = warehouse_object_prefix + .map(|prefix| format!("{prefix}{DATA_DIR}")) + .unwrap_or_else(|| default_table_data_dir_path(namespace, table)); + let default_data_prefix = format!("{}/", default_table_data_dir_path(namespace, table)); + if let Some(relative_path) = location.strip_prefix(&default_data_prefix) { + return Some(compaction_data_file_output_prefix(&warehouse_data_prefix, relative_path)); + } + if let Some(warehouse_object_prefix) = warehouse_object_prefix { + let warehouse_input_prefix = format!("{warehouse_object_prefix}{DATA_DIR}/"); + if let Some(relative_path) = location.strip_prefix(&warehouse_input_prefix) { + return Some(compaction_data_file_output_prefix(&warehouse_data_prefix, relative_path)); + } + } + None +} + +fn compaction_data_file_output_prefix(output_data_prefix: &str, relative_path: &str) -> String { + relative_path + .rsplit_once('/') + .map(|(partition_path, _)| format!("{output_data_prefix}/{partition_path}")) + .unwrap_or_else(|| output_data_prefix.to_string()) +} + +fn push_compaction_rewrite_group( + groups: &mut Vec, + current_locations: &mut Vec, + current_bytes: &mut u64, + sort_order_id: Option, + config: &TableCompactionPlanningConfig, +) { + if current_locations.len() >= config.min_input_files { + let input_file_count = current_locations.len(); + groups.push(TableCompactionRewriteGroup { + group_id: format!("{:04}", groups.len() + 1), + sort_order_id, + input_file_locations: std::mem::take(current_locations), + input_file_count, + input_bytes: *current_bytes, + output_file_location: None, + output_bytes: None, + }); + } else { + current_locations.clear(); + } + *current_bytes = 0; +} + +pub(crate) fn compaction_rewrite_group_partition( + data_files_by_key: &BTreeMap<&str, &CompactedDataFile>, + rewrite_group: &TableCompactionRewriteGroup, +) -> TableCatalogStoreResult<(i32, Vec<(String, apache_avro::types::Value)>)> { + let mut partition_spec_id = None; + let mut partition = None; + for input in &rewrite_group.input_file_locations { + let Some(data_file) = data_files_by_key.get(input.as_str()) else { + return Err(TableCatalogStoreError::Invalid( + "compaction rewrite input is missing from current manifest".to_string(), + )); + }; + match (partition_spec_id, partition.as_ref()) { + (None, None) => { + partition_spec_id = Some(data_file.partition_spec_id); + partition = Some(data_file.partition.clone()); + } + (Some(expected_spec_id), Some(expected_partition)) + if expected_spec_id == data_file.partition_spec_id && expected_partition == &data_file.partition => {} + _ => { + return Err(TableCatalogStoreError::Invalid( + "compaction rewrite group must contain a single partition tuple".to_string(), + )); + } + } + } + Ok((partition_spec_id.unwrap_or(0), partition.unwrap_or_default())) +} + +pub(crate) fn compaction_rewrite_group_sort_order( + data_files_by_key: &BTreeMap<&str, &CompactedDataFile>, + rewrite_group: &TableCompactionRewriteGroup, +) -> TableCatalogStoreResult> { + let mut sort_order_id = None; + let mut initialized = false; + for input in &rewrite_group.input_file_locations { + let Some(data_file) = data_files_by_key.get(input.as_str()) else { + return Err(TableCatalogStoreError::Invalid( + "compaction rewrite input is missing from current manifest".to_string(), + )); + }; + if !initialized { + sort_order_id = data_file.sort_order_id; + initialized = true; + } else if sort_order_id != data_file.sort_order_id { + return Err(TableCatalogStoreError::Invalid( + "compaction rewrite group must contain a single sort order".to_string(), + )); + } + } + if rewrite_group.sort_order_id != sort_order_id { + return Err(TableCatalogStoreError::Invalid( + "compaction rewrite group sort order changed after planning".to_string(), + )); + } + Ok(sort_order_id) +} + +pub(crate) fn compaction_manifest_partition_spec_id(data_files: &[CompactedDataFile]) -> TableCatalogStoreResult { + let Some(first) = data_files.first() else { + return Ok(0); + }; + let partition_spec_id = first.partition_spec_id; + if data_files + .iter() + .any(|data_file| data_file.partition_spec_id != partition_spec_id) + { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest cannot mix partition spec ids".to_string(), + )); + } + Ok(partition_spec_id) +} + +pub(crate) fn compact_parquet_data_files(input_files: &[(String, Vec)]) -> TableCatalogStoreResult { + let mut schema: Option = None; + let mut batches = Vec::new(); + let mut record_count = 0_u64; + + for (location, data) in input_files { + let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(data.clone())).map_err(|err| { + TableCatalogStoreError::Invalid(format!("failed to read compaction input parquet {location}: {err}")) + })?; + let file_schema = builder.schema().clone(); + match schema.as_ref() { + Some(expected_schema) if expected_schema.as_ref() != file_schema.as_ref() => { + return Err(TableCatalogStoreError::Invalid("compaction input parquet schemas must match".to_string())); + } + Some(_) => {} + None => schema = Some(file_schema), + } + + let reader = builder.build().map_err(|err| { + TableCatalogStoreError::Invalid(format!("failed to build compaction parquet reader {location}: {err}")) + })?; + for batch in reader { + let batch = batch.map_err(|err| { + TableCatalogStoreError::Invalid(format!("failed to read compaction parquet batch {location}: {err}")) + })?; + record_count = record_count.saturating_add(u64::try_from(batch.num_rows()).unwrap_or(u64::MAX)); + batches.push(batch); + } + } + + let Some(schema) = schema else { + return Err(TableCatalogStoreError::Invalid( + "compaction requires at least one parquet input file".to_string(), + )); + }; + let mut data = Vec::new(); + { + let mut writer = ArrowWriter::try_new(&mut data, schema, None) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to build compaction parquet writer: {err}")))?; + for batch in batches { + writer + .write(&batch) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to write compaction parquet batch: {err}")))?; + } + writer + .close() + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to close compaction parquet writer: {err}")))?; + } + + Ok(CompactedParquetFile { data, record_count }) +} + +fn validate_compaction_manifest_entry_status(entry_status: Option) -> TableCatalogStoreResult<()> { + match entry_status { + Some(0 | 1) => Ok(()), + Some(2) => Err(TableCatalogStoreError::Invalid( + "compaction currently does not support deleted manifest entries".to_string(), + )), + Some(_) => Err(TableCatalogStoreError::Invalid( + "compaction manifest entry status is unsupported".to_string(), + )), + None => Err(TableCatalogStoreError::Invalid("compaction manifest entry missing status".to_string())), + } +} + +fn parquet_record_count(data: &[u8]) -> TableCatalogStoreResult { + let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::copy_from_slice(data)) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read compaction parquet metadata: {err}")))?; + u64::try_from(builder.metadata().file_metadata().num_rows()) + .map_err(|_| TableCatalogStoreError::Invalid("compaction parquet record count must not be negative".to_string())) +} + +pub(crate) fn compacted_manifest_list_avro_bytes(summary: CompactionManifestListSummary<'_>) -> TableCatalogStoreResult> { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string"}, + {"name": "manifest_length", "type": "long"}, + {"name": "partition_spec_id", "type": "int"}, + {"name": "content", "type": "int"}, + {"name": "sequence_number", "type": "long"}, + {"name": "min_sequence_number", "type": "long"}, + {"name": "added_snapshot_id", "type": "long"}, + {"name": "added_files_count", "type": "int"}, + {"name": "existing_files_count", "type": "int"}, + {"name": "deleted_files_count", "type": "int"}, + {"name": "added_rows_count", "type": "long"}, + {"name": "existing_rows_count", "type": "long"}, + {"name": "deleted_rows_count", "type": "long"}, + {"name": "partitions", "type": ["null", {"type": "array", "items": {"type": "record", "name": "field_summary", "fields": [ + {"name": "contains_null", "type": "boolean"}, + {"name": "lower_bound", "type": ["null", "bytes"], "default": null}, + {"name": "upper_bound", "type": ["null", "bytes"], "default": null} + ]}}], "default": null} + ] + } + "#, + ) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to build compaction manifest list schema: {err}")))?; + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + writer + .append(apache_avro::types::Value::Record(vec![ + ( + "manifest_path".to_string(), + apache_avro::types::Value::String(summary.manifest_path.to_string()), + ), + ( + "manifest_length".to_string(), + apache_avro::types::Value::Long(i64::try_from(summary.manifest_length).unwrap_or(i64::MAX)), + ), + ("partition_spec_id".to_string(), apache_avro::types::Value::Int(summary.partition_spec_id)), + ("content".to_string(), apache_avro::types::Value::Int(0)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(summary.sequence_number)), + ( + "min_sequence_number".to_string(), + apache_avro::types::Value::Long(summary.sequence_number), + ), + ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(summary.snapshot_id)), + ( + "added_files_count".to_string(), + apache_avro::types::Value::Int(i32::try_from(summary.added_files_count).unwrap_or(i32::MAX)), + ), + ( + "existing_files_count".to_string(), + apache_avro::types::Value::Int(i32::try_from(summary.existing_files_count).unwrap_or(i32::MAX)), + ), + ("deleted_files_count".to_string(), apache_avro::types::Value::Int(0)), + ( + "added_rows_count".to_string(), + apache_avro::types::Value::Long(i64::try_from(summary.added_rows_count).unwrap_or(i64::MAX)), + ), + ( + "existing_rows_count".to_string(), + apache_avro::types::Value::Long(i64::try_from(summary.existing_rows_count).unwrap_or(i64::MAX)), + ), + ("deleted_rows_count".to_string(), apache_avro::types::Value::Long(0)), + ( + "partitions".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ])) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to write compaction manifest list: {err}")))?; + writer + .into_inner() + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to flush compaction manifest list: {err}"))) +} + +fn compacted_manifest_avro_schema(data_files: &[CompactedDataFile]) -> TableCatalogStoreResult { + let partition_fields = compaction_partition_schema_fields(data_files)?; + let partition_schema_fields = partition_fields + .into_iter() + .map(|(name, field_type)| { + serde_json::json!({ + "name": name, + "type": field_type + }) + }) + .collect::>(); + let schema = serde_json::json!({ + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": "long"}, + {"name": "sequence_number", "type": "long"}, + {"name": "file_sequence_number", "type": "long"}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "data_file", + "fields": [ + {"name": "content", "type": "int"}, + {"name": "file_path", "type": "string"}, + {"name": "file_format", "type": "string"}, + { + "name": "partition", + "type": { + "type": "record", + "name": "partition", + "fields": partition_schema_fields + } + }, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"}, + {"name": "column_sizes", "type": ["null", {"type": "map", "values": "long"}], "default": null}, + {"name": "value_counts", "type": ["null", {"type": "map", "values": "long"}], "default": null}, + {"name": "null_value_counts", "type": ["null", {"type": "map", "values": "long"}], "default": null}, + {"name": "nan_value_counts", "type": ["null", {"type": "map", "values": "long"}], "default": null}, + {"name": "lower_bounds", "type": ["null", {"type": "map", "values": "bytes"}], "default": null}, + {"name": "upper_bounds", "type": ["null", {"type": "map", "values": "bytes"}], "default": null}, + {"name": "key_metadata", "type": ["null", "bytes"], "default": null}, + {"name": "split_offsets", "type": ["null", {"type": "array", "items": "long"}], "default": null}, + {"name": "equality_ids", "type": ["null", {"type": "array", "items": "int"}], "default": null}, + {"name": "sort_order_id", "type": ["null", "int"], "default": null} + ] + } + } + ] + }); + apache_avro::Schema::parse_str(&schema.to_string()) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to build compaction manifest schema: {err}"))) +} + +fn compaction_partition_schema_fields( + data_files: &[CompactedDataFile], +) -> TableCatalogStoreResult> { + let Some(first) = data_files.first() else { + return Ok(Vec::new()); + }; + let mut expected = Vec::with_capacity(first.partition.len()); + for (field_name, field_value) in &first.partition { + let Some(field_type) = compaction_partition_field_schema(field_value) else { + return Err(TableCatalogStoreError::Invalid( + "compaction partition value type is unsupported".to_string(), + )); + }; + expected.push((field_name.clone(), field_type)); + } + + for data_file in data_files.iter().skip(1) { + if data_file.partition.len() != expected.len() { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest partition schemas must match".to_string(), + )); + } + for ((expected_name, expected_type), (field_name, field_value)) in expected.iter().zip(&data_file.partition) { + let Some(field_type) = compaction_partition_field_schema(field_value) else { + return Err(TableCatalogStoreError::Invalid( + "compaction partition value type is unsupported".to_string(), + )); + }; + if expected_name != field_name || expected_type != &field_type { + return Err(TableCatalogStoreError::Invalid( + "compaction manifest partition schemas must match".to_string(), + )); + } + } + } + Ok(expected) +} + +fn compaction_partition_field_schema(value: &apache_avro::types::Value) -> Option { + match avro_non_union_value(value) { + apache_avro::types::Value::Boolean(_) => Some(serde_json::json!("boolean")), + apache_avro::types::Value::Int(_) => Some(serde_json::json!("int")), + apache_avro::types::Value::Long(_) => Some(serde_json::json!("long")), + apache_avro::types::Value::Float(_) => Some(serde_json::json!("float")), + apache_avro::types::Value::Double(_) => Some(serde_json::json!("double")), + apache_avro::types::Value::Bytes(_) => Some(serde_json::json!("bytes")), + apache_avro::types::Value::String(_) => Some(serde_json::json!("string")), + apache_avro::types::Value::Date(_) => Some(serde_json::json!({"type": "int", "logicalType": "date"})), + apache_avro::types::Value::TimeMillis(_) => Some(serde_json::json!({"type": "int", "logicalType": "time-millis"})), + apache_avro::types::Value::TimeMicros(_) => Some(serde_json::json!({"type": "long", "logicalType": "time-micros"})), + apache_avro::types::Value::TimestampMillis(_) => { + Some(serde_json::json!({"type": "long", "logicalType": "timestamp-millis"})) + } + apache_avro::types::Value::TimestampMicros(_) => { + Some(serde_json::json!({"type": "long", "logicalType": "timestamp-micros"})) + } + apache_avro::types::Value::LocalTimestampMillis(_) => { + Some(serde_json::json!({"type": "long", "logicalType": "local-timestamp-millis"})) + } + apache_avro::types::Value::LocalTimestampMicros(_) => { + Some(serde_json::json!({"type": "long", "logicalType": "local-timestamp-micros"})) + } + apache_avro::types::Value::Uuid(_) => Some(serde_json::json!({"type": "string", "logicalType": "uuid"})), + _ => None, + } +} + +pub(crate) fn compacted_manifest_avro_bytes(data_files: &[CompactedDataFile]) -> TableCatalogStoreResult> { + let schema = compacted_manifest_avro_schema(data_files)?; + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + for data_file in data_files { + let sort_order_id = match data_file.sort_order_id { + Some(sort_order_id) => apache_avro::types::Value::Union(1, Box::new(apache_avro::types::Value::Int(sort_order_id))), + None => apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + }; + writer + .append(apache_avro::types::Value::Record(vec![ + ("status".to_string(), apache_avro::types::Value::Int(data_file.status)), + ("snapshot_id".to_string(), apache_avro::types::Value::Long(data_file.snapshot_id)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(data_file.sequence_number)), + ( + "file_sequence_number".to_string(), + apache_avro::types::Value::Long(data_file.file_sequence_number), + ), + ( + "data_file".to_string(), + apache_avro::types::Value::Record(vec![ + ("content".to_string(), apache_avro::types::Value::Int(0)), + ("file_path".to_string(), apache_avro::types::Value::String(data_file.file_path.clone())), + ("file_format".to_string(), apache_avro::types::Value::String("PARQUET".to_string())), + ("partition".to_string(), apache_avro::types::Value::Record(data_file.partition.clone())), + ( + "record_count".to_string(), + apache_avro::types::Value::Long(i64::try_from(data_file.record_count).unwrap_or(i64::MAX)), + ), + ( + "file_size_in_bytes".to_string(), + apache_avro::types::Value::Long(i64::try_from(data_file.file_size_bytes).unwrap_or(i64::MAX)), + ), + ( + "column_sizes".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "value_counts".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "null_value_counts".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "nan_value_counts".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "lower_bounds".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "upper_bounds".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "key_metadata".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "split_offsets".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "equality_ids".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ("sort_order_id".to_string(), sort_order_id), + ]), + ), + ])) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to write compaction manifest: {err}")))?; + } + writer + .into_inner() + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to flush compaction manifest: {err}"))) +} + +pub(crate) fn compaction_snapshot_id(current_metadata: &serde_json::Value, entry: &TableEntry, now: OffsetDateTime) -> i64 { + let generation = i64::try_from(entry.generation).unwrap_or(i64::MAX); + let mut snapshot_id = unix_timestamp_millis(now).saturating_mul(1000).saturating_add(generation); + let existing_snapshot_ids = current_metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64)) + .collect::>(); + while existing_snapshot_ids.contains(&snapshot_id) { + snapshot_id = snapshot_id.saturating_add(1); + } + snapshot_id +} + +pub(crate) fn next_compaction_sequence_number(current_metadata: &serde_json::Value) -> i64 { + current_metadata + .get("last-sequence-number") + .and_then(serde_json::Value::as_i64) + .unwrap_or(0) + .saturating_add(1) +} + +pub(crate) fn compaction_metadata_json( + current_metadata: &serde_json::Value, + entry: &TableEntry, + snapshot_id: i64, + sequence_number: i64, + manifest_list: &str, + previous_metadata_location: &str, + now: OffsetDateTime, +) -> TableCatalogStoreResult> { + let mut metadata = current_metadata.clone(); + let now_ms = unix_timestamp_millis(now); + let Some(metadata_object) = metadata.as_object_mut() else { + return Err(TableCatalogStoreError::Invalid( + "compaction metadata source must be a JSON object".to_string(), + )); + }; + metadata_object.insert("last-sequence-number".to_string(), serde_json::json!(sequence_number)); + metadata_object.insert("last-updated-ms".to_string(), serde_json::json!(now_ms)); + metadata_object.insert("current-snapshot-id".to_string(), serde_json::json!(snapshot_id)); + + let snapshots = metadata_object + .entry("snapshots".to_string()) + .or_insert_with(|| serde_json::json!([])); + let Some(snapshots) = snapshots.as_array_mut() else { + return Err(TableCatalogStoreError::Invalid( + "compaction metadata snapshots must be an array".to_string(), + )); + }; + snapshots.push(serde_json::json!({ + "snapshot-id": snapshot_id, + "sequence-number": sequence_number, + "timestamp-ms": now_ms, + "manifest-list": manifest_list, + "summary": { + "operation": "rewrite", + "rustfs.maintenance": "compaction" + } + })); + + let snapshot_log = metadata_object + .entry("snapshot-log".to_string()) + .or_insert_with(|| serde_json::json!([])); + let Some(snapshot_log) = snapshot_log.as_array_mut() else { + return Err(TableCatalogStoreError::Invalid( + "compaction metadata snapshot log must be an array".to_string(), + )); + }; + snapshot_log.push(serde_json::json!({ + "timestamp-ms": now_ms, + "snapshot-id": snapshot_id + })); + + let metadata_log = metadata_object + .entry("metadata-log".to_string()) + .or_insert_with(|| serde_json::json!([])); + let Some(metadata_log) = metadata_log.as_array_mut() else { + return Err(TableCatalogStoreError::Invalid("compaction metadata log must be an array".to_string())); + }; + metadata_log.push(serde_json::json!({ + "timestamp-ms": now_ms, + "metadata-file": previous_metadata_location + })); + + let refs = metadata_object + .entry("refs".to_string()) + .or_insert_with(|| serde_json::json!({})); + let Some(refs) = refs.as_object_mut() else { + return Err(TableCatalogStoreError::Invalid("compaction metadata refs must be an object".to_string())); + }; + refs.insert( + ICEBERG_MAIN_REF.to_string(), + serde_json::json!({ + "snapshot-id": snapshot_id, + "type": "branch" + }), + ); + metadata_object + .entry("location".to_string()) + .or_insert_with(|| serde_json::json!(entry.warehouse_location)); + + serde_json::to_vec(&metadata) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize compaction metadata: {err}"))) +} + +fn snapshot_expiration_drafts( + current_metadata: &serde_json::Value, + current_snapshot_id: Option, +) -> Vec { + let Some(snapshots) = current_metadata.get("snapshots").and_then(serde_json::Value::as_array) else { + return Vec::new(); + }; + + snapshots + .iter() + .map(|snapshot| { + let snapshot_id = snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64); + let timestamp_ms = snapshot.get("timestamp-ms").and_then(serde_json::Value::as_i64); + let mut reasons = BTreeSet::new(); + if snapshot_id.is_none() { + reasons.insert(TableSnapshotExpirationReason::MissingSnapshotId); + } + if timestamp_ms.is_none() { + reasons.insert(TableSnapshotExpirationReason::MissingSnapshotTimestamp); + } + if snapshot_id.is_some() && snapshot_id == current_snapshot_id { + reasons.insert(TableSnapshotExpirationReason::CurrentSnapshot); + } + + TableSnapshotExpirationDraft { + snapshot_id, + sequence_number: snapshot.get("sequence-number").and_then(serde_json::Value::as_i64), + timestamp_ms, + manifest_list: snapshot + .get("manifest-list") + .and_then(serde_json::Value::as_str) + .map(ToString::to_string), + reasons, + } + }) + .collect() +} + +fn mark_recent_snapshots_to_keep(drafts: &mut [TableSnapshotExpirationDraft], min_snapshots_to_keep: usize) { + let mut snapshots_by_time = drafts + .iter() + .enumerate() + .filter_map(|(index, draft)| Some((draft.timestamp_ms?, index))) + .collect::>(); + snapshots_by_time.sort_by(|(left_timestamp, left_index), (right_timestamp, right_index)| { + right_timestamp.cmp(left_timestamp).then_with(|| left_index.cmp(right_index)) + }); + + for (_, index) in snapshots_by_time.into_iter().take(min_snapshots_to_keep) { + drafts[index] + .reasons + .insert(TableSnapshotExpirationReason::MinSnapshotsToKeep); + } +} + +fn snapshot_expiration_ref_state( + current_metadata: &serde_json::Value, + current_snapshot_id: Option, +) -> (BTreeSet, BTreeSet, BTreeSet) { + let mut protected_ref_snapshot_ids = BTreeSet::new(); + let mut user_defined_ref_snapshot_ids = BTreeSet::new(); + let mut ref_retention_conflict_snapshot_ids = BTreeSet::new(); + let Some(refs) = current_metadata.get("refs").and_then(serde_json::Value::as_object) else { + return ( + protected_ref_snapshot_ids, + user_defined_ref_snapshot_ids, + ref_retention_conflict_snapshot_ids, + ); + }; + + for (name, reference) in refs { + let Some(snapshot_id) = reference.get("snapshot-id").and_then(serde_json::Value::as_i64) else { + continue; + }; + if name != ICEBERG_MAIN_REF || Some(snapshot_id) != current_snapshot_id { + protected_ref_snapshot_ids.insert(snapshot_id); + } + if name != ICEBERG_MAIN_REF { + user_defined_ref_snapshot_ids.insert(snapshot_id); + } + if snapshot_ref_has_retention_policy(reference) { + ref_retention_conflict_snapshot_ids.insert(snapshot_id); + } + } + + ( + protected_ref_snapshot_ids, + user_defined_ref_snapshot_ids, + ref_retention_conflict_snapshot_ids, + ) +} + +fn snapshot_ref_has_retention_policy(reference: &serde_json::Value) -> bool { + reference.get(ICEBERG_REF_MIN_SNAPSHOTS_TO_KEEP_FIELD).is_some() + || reference.get(ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD).is_some() + || reference.get(ICEBERG_REF_MAX_REF_AGE_MS_FIELD).is_some() +} + +fn snapshot_expiration_table_property_conflicts( + current_metadata: &serde_json::Value, + config: &TableSnapshotExpirationConfig, +) -> bool { + let Some(properties) = current_metadata.get("properties").and_then(serde_json::Value::as_object) else { + return false; + }; + + if properties.contains_key(ICEBERG_MAX_REF_AGE_MS_PROPERTY) { + return true; + } + if retention_property_conflicts_usize(properties, ICEBERG_MIN_SNAPSHOTS_TO_KEEP_PROPERTY, config.min_snapshots_to_keep) { + return true; + } + retention_property_conflicts_i64(properties, ICEBERG_MAX_SNAPSHOT_AGE_MS_PROPERTY, config.max_snapshot_age_ms) +} + +fn retention_property_conflicts_usize( + properties: &serde_json::Map, + key: &str, + expected: usize, +) -> bool { + let Some(value) = properties.get(key) else { + return false; + }; + serde_json_i64(value).and_then(|value| usize::try_from(value).ok()) != Some(expected) +} + +fn retention_property_conflicts_i64(properties: &serde_json::Map, key: &str, expected: i64) -> bool { + let Some(value) = properties.get(key) else { + return false; + }; + serde_json_i64(value) != Some(expected) +} + +fn serde_json_i64(value: &serde_json::Value) -> Option { + value.as_i64().or_else(|| value.as_str()?.parse::().ok()) +} + +fn snapshot_expiration_requires_manual_review(reasons: &BTreeSet) -> bool { + reasons.contains(&TableSnapshotExpirationReason::MissingSnapshotId) + || reasons.contains(&TableSnapshotExpirationReason::MissingSnapshotTimestamp) + || reasons.contains(&TableSnapshotExpirationReason::UserDefinedSnapshotRef) + || reasons.contains(&TableSnapshotExpirationReason::SnapshotRefRetentionConflict) + || reasons.contains(&TableSnapshotExpirationReason::TableRetentionPropertyConflict) +} + +fn snapshot_expiration_is_retained(reasons: &BTreeSet) -> bool { + reasons.contains(&TableSnapshotExpirationReason::CurrentSnapshot) + || reasons.contains(&TableSnapshotExpirationReason::MinSnapshotsToKeep) + || reasons.contains(&TableSnapshotExpirationReason::ProtectedSnapshotRef) +} + +fn unix_timestamp_millis(now: OffsetDateTime) -> i64 { + now.unix_timestamp() + .saturating_mul(1000) + .saturating_add(i64::from(now.millisecond())) +} diff --git a/rustfs/src/table_catalog/maintenance/recovery.rs b/rustfs/src/table_catalog/maintenance/recovery.rs new file mode 100644 index 000000000..756b989d0 --- /dev/null +++ b/rustfs/src/table_catalog/maintenance/recovery.rs @@ -0,0 +1,716 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::*; + +pub(crate) fn insert_metadata_maintenance_reason( + reasons_by_location: &mut BTreeMap>, + metadata_location: String, + reason: TableMetadataMaintenanceReason, +) { + reasons_by_location.entry(metadata_location).or_default().insert(reason); +} + +pub(crate) fn metadata_maintenance_object_reports( + reasons_by_location: BTreeMap>, +) -> Vec { + reasons_by_location + .into_iter() + .map(|(metadata_location, reasons)| { + let reasons = reasons.into_iter().collect::>(); + let state = if reasons.contains(&TableMetadataMaintenanceReason::SafetyWindowSatisfied) { + TableMetadataMaintenanceObjectState::Deletable + } else if reasons.contains(&TableMetadataMaintenanceReason::SafetyWindowPending) { + TableMetadataMaintenanceObjectState::PendingSafetyWindow + } else { + TableMetadataMaintenanceObjectState::Retained + }; + TableMetadataMaintenanceObjectReport { + metadata_location, + state, + reasons, + } + }) + .collect() +} + +#[derive(Debug, Clone)] +struct TableMetadataMaintenanceReferencedObjectAccumulator { + object_kind: TableMetadataMaintenanceObjectKind, + state: TableMetadataMaintenanceObjectState, + reasons: BTreeSet, +} + +fn insert_referenced_object_report( + reports: &mut BTreeMap, + object_location: String, + object_kind: TableMetadataMaintenanceObjectKind, + state: TableMetadataMaintenanceObjectState, + reason: TableMetadataMaintenanceReason, +) { + let report = reports + .entry(object_location) + .or_insert_with(|| TableMetadataMaintenanceReferencedObjectAccumulator { + object_kind, + state: TableMetadataMaintenanceObjectState::Retained, + reasons: BTreeSet::new(), + }); + if state == TableMetadataMaintenanceObjectState::ManualReviewRequired { + report.state = TableMetadataMaintenanceObjectState::ManualReviewRequired; + } + report.reasons.insert(reason); +} + +pub(crate) async fn metadata_maintenance_referenced_object_reports( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + current_metadata: &serde_json::Value, + retained_metadata_locations: &[String], +) -> TableCatalogStoreResult> +where + B: TableCatalogObjectBackend, +{ + let mut reports = BTreeMap::::new(); + metadata_maintenance_referenced_object_reports_for_metadata( + backend, + table_bucket, + namespace, + table, + warehouse_object_prefix, + current_metadata, + &mut reports, + ) + .await?; + + for metadata_location in retained_metadata_locations { + let Some(metadata_object) = backend.read_object(table_bucket, metadata_location).await? else { + insert_referenced_object_report( + &mut reports, + metadata_location.clone(), + TableMetadataMaintenanceObjectKind::MetadataFile, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnreadableMetadata, + ); + continue; + }; + let Ok(metadata) = serde_json::from_slice::(&metadata_object.data) else { + insert_referenced_object_report( + &mut reports, + metadata_location.clone(), + TableMetadataMaintenanceObjectKind::MetadataFile, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnreadableMetadata, + ); + continue; + }; + if !metadata.is_object() { + insert_referenced_object_report( + &mut reports, + metadata_location.clone(), + TableMetadataMaintenanceObjectKind::MetadataFile, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnreadableMetadata, + ); + continue; + } + metadata_maintenance_referenced_object_reports_for_metadata( + backend, + table_bucket, + namespace, + table, + warehouse_object_prefix, + &metadata, + &mut reports, + ) + .await?; + } + + Ok(reports + .into_iter() + .map(|(object_location, report)| TableMetadataMaintenanceReferencedObjectReport { + object_location, + object_kind: report.object_kind, + state: report.state, + reasons: report.reasons.into_iter().collect(), + }) + .collect()) +} + +async fn metadata_maintenance_referenced_object_reports_for_metadata( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + metadata: &serde_json::Value, + reports: &mut BTreeMap, +) -> TableCatalogStoreResult<()> +where + B: TableCatalogObjectBackend, +{ + let Some(snapshots) = metadata.get("snapshots").and_then(serde_json::Value::as_array) else { + return Ok(()); + }; + + for snapshot in snapshots { + if let Some(manifest_list_location) = snapshot.get("manifest-list").and_then(serde_json::Value::as_str) { + metadata_maintenance_referenced_manifest_list( + backend, + table_bucket, + namespace, + table, + warehouse_object_prefix, + manifest_list_location, + reports, + ) + .await?; + continue; + } + + let Some(manifests) = snapshot.get("manifests").and_then(serde_json::Value::as_array) else { + continue; + }; + for manifest in manifests { + let Some(manifest_location) = manifest.as_str() else { + insert_referenced_object_report( + reports, + "snapshots[].manifests".to_string(), + TableMetadataMaintenanceObjectKind::ManifestFile, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + continue; + }; + metadata_maintenance_referenced_manifest_file( + backend, + table_bucket, + namespace, + table, + warehouse_object_prefix, + manifest_location, + reports, + ) + .await?; + } + } + + Ok(()) +} + +async fn metadata_maintenance_referenced_manifest_list( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + manifest_list_location: &str, + reports: &mut BTreeMap, +) -> TableCatalogStoreResult<()> +where + B: TableCatalogObjectBackend, +{ + let Some(manifest_list_key) = table_catalog_object_key_from_location(table_bucket, manifest_list_location) else { + insert_referenced_object_report( + reports, + manifest_list_location.to_string(), + TableMetadataMaintenanceObjectKind::ManifestList, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + return Ok(()); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &manifest_list_key) + != Some(TableMetadataMaintenanceObjectKind::ManifestList) + { + insert_referenced_object_report( + reports, + manifest_list_key, + TableMetadataMaintenanceObjectKind::ManifestList, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + return Ok(()); + } + insert_referenced_object_report( + reports, + manifest_list_key.clone(), + TableMetadataMaintenanceObjectKind::ManifestList, + TableMetadataMaintenanceObjectState::Retained, + TableMetadataMaintenanceReason::ManifestList, + ); + + let Some(manifest_list_object) = backend.read_object(table_bucket, &manifest_list_key).await? else { + mark_referenced_object_manual_review( + reports, + &manifest_list_key, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + return Ok(()); + }; + let Ok(manifest_paths) = manifest_paths_from_manifest_list_avro(&manifest_list_object.data) else { + mark_referenced_object_manual_review( + reports, + &manifest_list_key, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + return Ok(()); + }; + for manifest_location in manifest_paths { + metadata_maintenance_referenced_manifest_file( + backend, + table_bucket, + namespace, + table, + warehouse_object_prefix, + &manifest_location, + reports, + ) + .await?; + } + + Ok(()) +} + +async fn metadata_maintenance_referenced_manifest_file( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + manifest_location: &str, + reports: &mut BTreeMap, +) -> TableCatalogStoreResult<()> +where + B: TableCatalogObjectBackend, +{ + let Some(manifest_key) = table_catalog_object_key_from_location(table_bucket, manifest_location) else { + insert_referenced_object_report( + reports, + manifest_location.to_string(), + TableMetadataMaintenanceObjectKind::ManifestFile, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + return Ok(()); + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &manifest_key) + != Some(TableMetadataMaintenanceObjectKind::ManifestFile) + { + insert_referenced_object_report( + reports, + manifest_key, + TableMetadataMaintenanceObjectKind::ManifestFile, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + return Ok(()); + } + insert_referenced_object_report( + reports, + manifest_key.clone(), + TableMetadataMaintenanceObjectKind::ManifestFile, + TableMetadataMaintenanceObjectState::Retained, + TableMetadataMaintenanceReason::ManifestFile, + ); + + let Some(manifest_object) = backend.read_object(table_bucket, &manifest_key).await? else { + mark_referenced_object_manual_review(reports, &manifest_key, TableMetadataMaintenanceReason::UnsupportedManifestAvro); + return Ok(()); + }; + let Ok(file_references) = file_references_from_manifest_avro(&manifest_object.data) else { + mark_referenced_object_manual_review(reports, &manifest_key, TableMetadataMaintenanceReason::UnsupportedManifestAvro); + return Ok(()); + }; + for (file_location, object_kind) in file_references { + let Some(file_key) = table_catalog_object_key_from_location(table_bucket, &file_location) else { + insert_referenced_object_report( + reports, + file_location, + object_kind, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + continue; + }; + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &file_key) != Some(object_kind.clone()) { + insert_referenced_object_report( + reports, + file_key, + object_kind, + TableMetadataMaintenanceObjectState::ManualReviewRequired, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ); + continue; + } + insert_referenced_object_report( + reports, + file_key, + object_kind.clone(), + TableMetadataMaintenanceObjectState::Retained, + table_metadata_maintenance_reason_for_object_kind(&object_kind), + ); + } + + Ok(()) +} + +fn mark_referenced_object_manual_review( + reports: &mut BTreeMap, + object_location: &str, + reason: TableMetadataMaintenanceReason, +) { + if let Some(report) = reports.get_mut(object_location) { + report.state = TableMetadataMaintenanceObjectState::ManualReviewRequired; + report.reasons.insert(reason); + } +} + +pub(crate) fn table_catalog_object_key_from_location(table_bucket: &str, location: &str) -> Option { + let object = if let Some(location) = location.strip_prefix("s3://") { + let (bucket, object) = location.split_once('/')?; + if bucket != table_bucket { + return None; + } + object + } else { + location + }; + + if object.is_empty() + || object.starts_with('/') + || object.contains("..") + || object.contains('\\') + || object.bytes().any(|byte| byte.is_ascii_control()) + { + return None; + } + + Some(object.to_string()) +} + +pub(crate) fn table_maintenance_object_kind( + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + object_location: &str, +) -> Option { + let metadata_prefix = format!("{}/", default_table_metadata_dir_path(namespace, table)); + if let Some(kind) = table_maintenance_metadata_object_kind(&metadata_prefix, object_location) { + return Some(kind); + } + + let data_prefix = format!("{}/", default_table_data_dir_path(namespace, table)); + if object_location + .strip_prefix(&data_prefix) + .is_some_and(is_valid_table_maintenance_nested_object) + { + return Some(TableMetadataMaintenanceObjectKind::DataFile); + } + + let delete_prefix = format!("{}/", default_table_delete_dir_path(namespace, table)); + if object_location + .strip_prefix(&delete_prefix) + .is_some_and(is_valid_table_maintenance_nested_object) + { + return Some(TableMetadataMaintenanceObjectKind::DeleteFile); + } + + if let Some(warehouse_object_prefix) = warehouse_object_prefix { + let metadata_prefix = format!("{warehouse_object_prefix}{METADATA_DIR}/"); + if let Some(kind) = table_maintenance_metadata_object_kind(&metadata_prefix, object_location) { + return Some(kind); + } + + let data_prefix = format!("{warehouse_object_prefix}{DATA_DIR}/"); + if object_location + .strip_prefix(&data_prefix) + .is_some_and(is_valid_table_maintenance_nested_object) + { + return Some(TableMetadataMaintenanceObjectKind::DataFile); + } + + let delete_prefix = format!("{warehouse_object_prefix}{DELETE_DIR}/"); + if object_location + .strip_prefix(&delete_prefix) + .is_some_and(is_valid_table_maintenance_nested_object) + { + return Some(TableMetadataMaintenanceObjectKind::DeleteFile); + } + } + + None +} + +fn table_maintenance_metadata_object_kind( + metadata_prefix: &str, + object_location: &str, +) -> Option { + let file_name = object_location.strip_prefix(metadata_prefix)?; + if file_name.is_empty() + || file_name.contains('/') + || file_name.contains('\\') + || file_name.contains("..") + || file_name.bytes().any(|byte| byte.is_ascii_control()) + || !file_name.ends_with(".avro") + { + return None; + } + if file_name.starts_with("snap-") { + return Some(TableMetadataMaintenanceObjectKind::ManifestList); + } + Some(TableMetadataMaintenanceObjectKind::ManifestFile) +} + +fn is_valid_table_maintenance_nested_object(suffix: &str) -> bool { + !suffix.is_empty() + && !suffix.starts_with('/') + && !suffix.contains("..") + && !suffix.contains('\\') + && !suffix.bytes().any(|byte| byte.is_ascii_control()) +} + +fn table_metadata_maintenance_reason_for_object_kind( + object_kind: &TableMetadataMaintenanceObjectKind, +) -> TableMetadataMaintenanceReason { + match object_kind { + TableMetadataMaintenanceObjectKind::MetadataFile => TableMetadataMaintenanceReason::CurrentMetadata, + TableMetadataMaintenanceObjectKind::ManifestList => TableMetadataMaintenanceReason::ManifestList, + TableMetadataMaintenanceObjectKind::ManifestFile => TableMetadataMaintenanceReason::ManifestFile, + TableMetadataMaintenanceObjectKind::DataFile => TableMetadataMaintenanceReason::DataFile, + TableMetadataMaintenanceObjectKind::DeleteFile => TableMetadataMaintenanceReason::DeleteFile, + } +} + +pub(crate) fn metadata_maintenance_reachability_graph_report( + metadata_file_count: usize, + referenced_object_reports: &[TableMetadataMaintenanceReferencedObjectReport], +) -> TableMaintenanceReachabilityGraphReport { + let manifest_list_count = referenced_object_reports + .iter() + .filter(|report| report.object_kind == TableMetadataMaintenanceObjectKind::ManifestList) + .count(); + let manifest_file_count = referenced_object_reports + .iter() + .filter(|report| report.object_kind == TableMetadataMaintenanceObjectKind::ManifestFile) + .count(); + let data_file_count = referenced_object_reports + .iter() + .filter(|report| report.object_kind == TableMetadataMaintenanceObjectKind::DataFile) + .count(); + let delete_file_count = referenced_object_reports + .iter() + .filter(|report| report.object_kind == TableMetadataMaintenanceObjectKind::DeleteFile) + .count(); + let manual_review_count = referenced_object_reports + .iter() + .filter(|report| report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired) + .count(); + let mut reasons = BTreeSet::from([TableMaintenanceReachabilityGraphReason::MetadataJsonParsed]); + if manifest_list_count > 0 { + reasons.insert(TableMaintenanceReachabilityGraphReason::ManifestListAvroReferenced); + } + if referenced_object_reports.iter().any(|report| { + report + .reasons + .contains(&TableMetadataMaintenanceReason::UnsupportedManifestAvro) + }) { + reasons.insert(TableMaintenanceReachabilityGraphReason::ManifestAvroReaderUnavailable); + } + + TableMaintenanceReachabilityGraphReport { + status: if manual_review_count == 0 { + TableMaintenanceReachabilityGraphStatus::Complete + } else { + TableMaintenanceReachabilityGraphStatus::ManualReviewRequired + }, + metadata_file_count, + manifest_list_count, + manifest_file_count, + data_file_count, + delete_file_count, + manual_review_count, + reasons: reasons.into_iter().collect(), + } +} + +pub(crate) async fn metadata_maintenance_object_cleanup_reports( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, + referenced_object_reports: &[TableMetadataMaintenanceReferencedObjectReport], + now: OffsetDateTime, +) -> TableCatalogStoreResult<(usize, Vec, Vec, Vec)> +where + B: TableCatalogObjectBackend, +{ + let scanned_objects = + table_maintenance_cleanup_objects(backend, table_bucket, namespace, table, warehouse_object_prefix).await?; + if referenced_object_reports + .iter() + .any(|report| report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired) + { + return Ok((scanned_objects.len(), Vec::new(), Vec::new(), Vec::new())); + } + + let referenced_locations = referenced_object_reports + .iter() + .filter_map(|report| table_catalog_object_key_from_location(table_bucket, &report.object_location)) + .collect::>(); + let mut cleanup_candidate_locations = Vec::new(); + let mut deletable_object_locations = Vec::new(); + let mut cleanup_reports = Vec::new(); + + for (object_location, object_kind) in scanned_objects { + if referenced_locations.contains(&object_location) { + continue; + } + let mut reasons = BTreeSet::from([ + table_metadata_maintenance_reason_for_object_kind(&object_kind), + TableMetadataMaintenanceReason::NoCurrentReachability, + ]); + let state = match backend.read_object(table_bucket, &object_location).await? { + Some(object) if metadata_candidate_is_past_safety_window(object.mod_time, now) => { + reasons.insert(TableMetadataMaintenanceReason::SafetyWindowSatisfied); + cleanup_candidate_locations.push(object_location.clone()); + deletable_object_locations.push(object_location.clone()); + TableMetadataMaintenanceObjectState::Deletable + } + _ => { + reasons.insert(TableMetadataMaintenanceReason::SafetyWindowPending); + cleanup_candidate_locations.push(object_location.clone()); + TableMetadataMaintenanceObjectState::PendingSafetyWindow + } + }; + cleanup_reports.push(TableMetadataMaintenanceObjectCleanupReport { + object_location, + object_kind, + state, + reasons: reasons.into_iter().collect(), + }); + } + + Ok(( + referenced_locations.len() + cleanup_reports.len(), + cleanup_candidate_locations, + deletable_object_locations, + cleanup_reports, + )) +} + +async fn table_maintenance_cleanup_objects( + backend: &B, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + warehouse_object_prefix: Option<&str>, +) -> TableCatalogStoreResult> +where + B: TableCatalogObjectBackend, +{ + let mut objects = BTreeMap::new(); + let mut metadata_prefixes = vec![format!("{}/", default_table_metadata_dir_path(namespace, table))]; + let mut data_prefixes = vec![format!("{}/", default_table_data_dir_path(namespace, table))]; + let mut delete_prefixes = vec![format!("{}/", default_table_delete_dir_path(namespace, table))]; + if let Some(warehouse_object_prefix) = warehouse_object_prefix { + metadata_prefixes.push(format!("{warehouse_object_prefix}{METADATA_DIR}/")); + data_prefixes.push(format!("{warehouse_object_prefix}{DATA_DIR}/")); + delete_prefixes.push(format!("{warehouse_object_prefix}{DELETE_DIR}/")); + } + metadata_prefixes.sort(); + metadata_prefixes.dedup(); + data_prefixes.sort(); + data_prefixes.dedup(); + delete_prefixes.sort(); + delete_prefixes.dedup(); + + for metadata_prefix in metadata_prefixes { + for object in backend.list_objects(table_bucket, &metadata_prefix).await? { + if let Some(kind) = table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &object) + && matches!( + kind, + TableMetadataMaintenanceObjectKind::ManifestList | TableMetadataMaintenanceObjectKind::ManifestFile + ) + { + objects.insert(object, kind); + } + } + } + + for data_prefix in data_prefixes { + for object in backend.list_objects(table_bucket, &data_prefix).await? { + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &object) + == Some(TableMetadataMaintenanceObjectKind::DataFile) + { + objects.insert(object, TableMetadataMaintenanceObjectKind::DataFile); + } + } + } + + for delete_prefix in delete_prefixes { + for object in backend.list_objects(table_bucket, &delete_prefix).await? { + if table_maintenance_object_kind(namespace, table, warehouse_object_prefix, &object) + == Some(TableMetadataMaintenanceObjectKind::DeleteFile) + { + objects.insert(object, TableMetadataMaintenanceObjectKind::DeleteFile); + } + } + } + + Ok(objects) +} + +pub(crate) fn mark_deleted_metadata_object_reports( + object_reports: &mut [TableMetadataMaintenanceObjectReport], + deleted_locations: &BTreeSet, +) { + for object_report in object_reports { + if !deleted_locations.contains(&object_report.metadata_location) { + continue; + } + object_report.state = TableMetadataMaintenanceObjectState::Deleted; + if !object_report + .reasons + .contains(&TableMetadataMaintenanceReason::DeletedByMaintenance) + { + object_report + .reasons + .push(TableMetadataMaintenanceReason::DeletedByMaintenance); + } + } +} + +pub(crate) fn mark_deleted_object_cleanup_reports( + object_reports: &mut [TableMetadataMaintenanceObjectCleanupReport], + deleted_locations: &BTreeSet, +) { + for object_report in object_reports { + if !deleted_locations.contains(&object_report.object_location) { + continue; + } + object_report.state = TableMetadataMaintenanceObjectState::Deleted; + if !object_report + .reasons + .contains(&TableMetadataMaintenanceReason::DeletedByMaintenance) + { + object_report + .reasons + .push(TableMetadataMaintenanceReason::DeletedByMaintenance); + } + } +} diff --git a/rustfs/src/table_catalog/maintenance/worker.rs b/rustfs/src/table_catalog/maintenance/worker.rs new file mode 100644 index 000000000..7c75aa413 --- /dev/null +++ b/rustfs/src/table_catalog/maintenance/worker.rs @@ -0,0 +1,225 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::*; + +pub(crate) fn maintenance_timestamp(now: OffsetDateTime) -> String { + now.format(&time::format_description::well_known::Rfc3339) + .unwrap_or_else(|_| now.unix_timestamp().to_string()) +} + +pub(crate) fn default_table_maintenance_worker_lease_timeout_seconds() -> u64 { + TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_DEFAULT_SECONDS +} + +fn parse_maintenance_timestamp(timestamp: &str) -> Option { + OffsetDateTime::parse(timestamp, &time::format_description::well_known::Rfc3339).ok() +} + +pub(crate) fn table_maintenance_quarantine_operator_reason(action: &str, reason: Option<&str>) -> String { + let reason = reason.map(str::trim).filter(|reason| !reason.is_empty()); + match reason { + Some(reason) => format!("maintenance quarantine {action} by operator: {reason}"), + None => format!("maintenance quarantine {action} by operator"), + } +} + +pub(crate) fn push_table_maintenance_audit_event( + report: &mut TableMetadataMaintenanceReport, + timestamp: OffsetDateTime, + actor: TableMaintenanceAuditActor, + action: TableMaintenanceAuditAction, + reason: Option, + before_status: Option, + before_quarantined_object_count: Option, +) { + report.audit_events.push(TableMaintenanceAuditEvent { + timestamp: maintenance_timestamp(timestamp), + actor, + action, + reason, + before_status, + after_status: Some(report.job.status.clone()), + before_quarantined_object_count, + after_quarantined_object_count: Some(report.job.quarantined_object_count), + recommended_actions: report.job.recommended_actions.clone(), + }); +} + +fn table_maintenance_recommended_actions(job: &TableMetadataMaintenanceJob) -> Vec { + let mut actions = Vec::new(); + match job.status { + TableMetadataMaintenanceJobStatus::NotYetRun => {} + TableMetadataMaintenanceJobStatus::Queued => { + actions.push(TableMaintenanceRecommendedAction::RunMaintenanceWorker); + } + TableMetadataMaintenanceJobStatus::Running => { + actions.push(TableMaintenanceRecommendedAction::WaitForActiveWorker); + } + TableMetadataMaintenanceJobStatus::Successful => { + if matches!(job.operation, TableMetadataMaintenanceOperation::DryRun) + && (job.deletable_metadata_file_count > 0 || job.deletable_object_count > 0) + { + actions.push(TableMaintenanceRecommendedAction::ReviewAndRunDelete); + } else { + actions.push(TableMaintenanceRecommendedAction::NoActionRequired); + } + } + TableMetadataMaintenanceJobStatus::Failed => { + if job + .failure_reason + .as_deref() + .is_some_and(|reason| reason == TABLE_MAINTENANCE_DELETE_DISABLED_REASON) + { + actions.push(TableMaintenanceRecommendedAction::EnableDelete); + } + if job.quarantine_enabled && job.quarantined_object_count > 0 { + actions.push(TableMaintenanceRecommendedAction::ReviewQuarantine); + } + if job.next_retry_after.is_some() { + actions.push(TableMaintenanceRecommendedAction::WaitForRetryBackoff); + } + if actions.is_empty() { + actions.push(TableMaintenanceRecommendedAction::InvestigateFailure); + } + } + TableMetadataMaintenanceJobStatus::Disabled => { + actions.push(TableMaintenanceRecommendedAction::EnableBackgroundMaintenance); + } + TableMetadataMaintenanceJobStatus::Paused => { + actions.push(TableMaintenanceRecommendedAction::ResumeMaintenanceWorker); + } + } + actions +} + +pub(crate) fn push_unique_maintenance_action( + actions: &mut Vec, + action: TableMaintenanceRecommendedAction, +) { + if !actions.contains(&action) { + actions.push(action); + } +} + +pub(crate) fn table_maintenance_report_order_timestamp(report: &TableMetadataMaintenanceReport) -> String { + report + .job + .finished_at + .clone() + .or_else(|| report.job.heartbeat_at.clone()) + .or_else(|| report.job.started_at.clone()) + .or_else(|| report.job.scheduled_at.clone()) + .unwrap_or_default() +} + +pub(crate) fn table_maintenance_scheduler_job_summary( + report: &TableMetadataMaintenanceReport, +) -> TableMaintenanceSchedulerJobSummary { + TableMaintenanceSchedulerJobSummary { + job_id: report.job.job_id.clone(), + operation: report.job.operation.clone(), + status: report.job.status.clone(), + scheduler_id: report.job.scheduler_id.clone(), + scheduled_at: report.job.scheduled_at.clone(), + worker_id: report.job.worker_id.clone(), + attempt: report.job.attempt, + started_at: report.job.started_at.clone(), + finished_at: report.job.finished_at.clone(), + heartbeat_at: report.job.heartbeat_at.clone(), + next_retry_after: report.job.next_retry_after.clone(), + recommended_actions: report.job.recommended_actions.clone(), + audit_events: report.audit_events.clone(), + } +} + +pub(crate) fn table_maintenance_scheduler_quarantine_boundary( + config: &TableMaintenanceConfig, + reports: &[TableMetadataMaintenanceReport], +) -> TableMaintenanceSchedulerQuarantineBoundary { + let source = reports + .iter() + .find(|report| report.job.quarantine_enabled && report.job.quarantined_object_count > 0); + TableMaintenanceSchedulerQuarantineBoundary { + enabled: config.quarantine_enabled, + active: source.is_some(), + retention_seconds: source.map_or(config.quarantine_retention_seconds, |report| report.job.quarantine_retention_seconds), + quarantined_object_count: source.map_or(0, |report| report.job.quarantined_object_count), + source_job_id: source.map(|report| report.job.job_id.clone()), + } +} + +pub(crate) fn refresh_table_maintenance_report_recommended_actions(report: &mut TableMetadataMaintenanceReport) { + report.job.recommended_actions = table_maintenance_recommended_actions(&report.job); +} + +pub(crate) fn table_maintenance_report_with_recommended_actions( + mut report: TableMetadataMaintenanceReport, +) -> TableMetadataMaintenanceReport { + refresh_table_maintenance_report_recommended_actions(&mut report); + report +} + +pub(crate) fn table_maintenance_scheduler_lease_is_active( + job: &TableMetadataMaintenanceJob, + scheduler_lease_timeout_seconds: u64, + now: OffsetDateTime, +) -> bool { + let Some(scheduled_at) = job.scheduled_at.as_deref().and_then(parse_maintenance_timestamp) else { + return false; + }; + let timeout_seconds = i64::try_from(scheduler_lease_timeout_seconds).unwrap_or(i64::MAX); + scheduled_at.saturating_add(Duration::seconds(timeout_seconds)) > now +} + +pub(crate) fn table_maintenance_job_lease_is_active( + job: &TableMetadataMaintenanceJob, + worker_lease_timeout_seconds: u64, + now: OffsetDateTime, +) -> bool { + let Some(heartbeat_at) = job.heartbeat_at.as_deref().and_then(parse_maintenance_timestamp) else { + return false; + }; + let timeout_seconds = i64::try_from(worker_lease_timeout_seconds).unwrap_or(i64::MAX); + heartbeat_at.saturating_add(Duration::seconds(timeout_seconds)) > now +} + +pub(crate) fn table_maintenance_job_retry_is_pending(job: &TableMetadataMaintenanceJob, now: OffsetDateTime) -> bool { + if !matches!(job.status, TableMetadataMaintenanceJobStatus::Failed) { + return false; + } + let Some(next_retry_after) = job.next_retry_after.as_deref().and_then(parse_maintenance_timestamp) else { + return false; + }; + next_retry_after > now +} + +pub(crate) fn apply_maintenance_retry_after( + job: &mut TableMetadataMaintenanceJob, + config: &TableMaintenanceConfig, + now: OffsetDateTime, +) { + if config.max_retry_attempts == 0 || job.attempt >= config.max_retry_attempts { + job.next_retry_after = None; + return; + } + let attempt_index = u32::from(job.attempt.saturating_sub(1)); + let multiplier = 1_u64.checked_shl(attempt_index).unwrap_or(u64::MAX); + let delay_seconds = config + .retry_initial_backoff_seconds + .saturating_mul(multiplier) + .min(config.retry_max_backoff_seconds); + let delay_seconds = i64::try_from(delay_seconds).unwrap_or(i64::MAX); + job.next_retry_after = Some(maintenance_timestamp(now.saturating_add(Duration::seconds(delay_seconds)))); +} diff --git a/rustfs/src/table_catalog/mod.rs b/rustfs/src/table_catalog/mod.rs new file mode 100644 index 000000000..b15ab9ef6 --- /dev/null +++ b/rustfs/src/table_catalog/mod.rs @@ -0,0 +1,345 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Internal table catalog primitives for the Iceberg REST Catalog framework. +//! +//! This module intentionally does not expose HTTP handlers or mutate existing +//! S3 object behavior. It defines the stable internal boundary that later +//! catalog routes and object guards can share. + +#![allow(dead_code)] + +use std::{ + collections::{BTreeMap, BTreeSet}, + num::NonZeroUsize, + ops::Bound, + sync::Arc, + time::{Duration as StdDuration, Instant}, +}; + +use crate::storage_api::table::contract::http::HTTPPreconditions; +use crate::storage_api::table::contract::list::{ + ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, + ListOperations as StorageListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions, +}; +use crate::storage_api::table::contract::namespace::NamespaceLocking as StorageNamespaceLocking; +use crate::storage_api::table::contract::object::{ObjectIO as StorageObjectIO, ObjectOperations as StorageObjectOperations}; +use crate::storage_api::table::contract::range::HTTPRangeSpec; +use crate::storage_api::table::{ + BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, + BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, RUSTFS_META_BUCKET, StorageError, get_bucket_metadata, + get_lock_acquire_timeout, table_catalog_path_hash, +}; +use bytes::Bytes; +use datafusion::{ + arrow::datatypes::SchemaRef, + parquet::arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder}, +}; +use http::HeaderMap; +use metrics::{counter, histogram}; +use rustfs_filemeta::FileInfo; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use sha2::{Digest, Sha256}; +use time::{Duration, OffsetDateTime}; +use tokio::io::AsyncReadExt; +use uuid::Uuid; + +use crate::storage_api::table::{ + StorageDeletedObject as DeletedObject, StorageGetObjectReader as GetObjectReader, StorageObjectInfo as ObjectInfo, + StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader, +}; + +mod error; +mod iceberg; +mod identifier; +mod maintenance; +mod model; +mod store; + +pub use error::{CatalogIdentifierError, TableObjectMutationError}; +pub(crate) use error::{TableCatalogStoreError, TableCatalogStoreResult}; +pub(crate) use iceberg::*; +pub use identifier::{IdentifierSegment, Namespace, is_reserved_table_object_key}; +pub(crate) use identifier::{ + default_table_data_dir_path, default_table_delete_dir_path, default_table_metadata_dir_path, + default_table_metadata_file_path, default_view_metadata_file_path, is_valid_table_metadata_location, + is_valid_view_metadata_location, metadata_location_from_metadata_file_path, validate_bucket_object_mutation, +}; +pub(crate) use maintenance::*; +pub(crate) use model::*; +pub(crate) use store::*; + +pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = BUCKET_TABLE_CONFIG; +pub(crate) const RESERVED_CATALOG_OBJECT_MESSAGE: &str = "Object key is reserved for the table catalog"; +pub(crate) const TABLE_BUCKET_CATALOG_TYPE: &str = "iceberg-rest"; +pub(crate) const TABLE_BUCKET_CONFIG_VERSION: u16 = 1; +pub(crate) const DEFAULT_WAREHOUSE_ID: &str = "default"; +pub(crate) const TABLE_NAMESPACE_MARKER_VERSION: u16 = 1; +pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1; +pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1; +pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1; +pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1; +pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1; +pub(crate) const TABLE_CATALOG_BACKING_MANIFEST_VERSION: u16 = 1; +pub(crate) const ENV_TABLE_CATALOG_BACKING: &str = "RUSTFS_TABLE_CATALOG_BACKING"; +pub(crate) const TABLE_CATALOG_BACKING_OBJECT: &str = "object"; +pub(crate) const TABLE_CATALOG_BACKING_DURABLE_STRONG: &str = "durable-strong"; +pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128; +pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX; +const WAREHOUSE_ROOT: &str = "warehouses"; +const NAMESPACE_ROOT: &str = "namespaces"; +const TABLE_ROOT: &str = "tables"; +const VIEW_ROOT: &str = "views"; +const NAMESPACE_MARKER_FILE: &str = "namespace.json"; +const TABLE_MARKER_FILE: &str = "table.json"; +const CURRENT_POINTER_FILE: &str = "current.json"; +const LIFECYCLE_FILE: &str = "lifecycle.json"; +const METADATA_DIR: &str = "metadata"; +const DATA_DIR: &str = "data"; +const DELETE_DIR: &str = "delete"; +const TABLE_BUCKET_ENTRY_FILE: &str = "table-bucket.json"; +const NAMESPACE_ENTRY_FILE: &str = "namespace-entry.json"; +const TABLE_ENTRY_FILE: &str = "table-entry.json"; +const VIEW_ENTRY_FILE: &str = "view-entry.json"; +const INTERNAL_CATALOG_ROOT: &str = BUCKET_TABLE_CATALOG_META_PREFIX; +const TABLE_BUCKET_ROOT: &str = BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX; +const COMMIT_LOG_ROOT: &str = "commits"; +const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency"; +const WAREHOUSE_INDEX_ROOT: &str = "warehouse-index"; +const WAREHOUSE_INDEX_STATE_FILE: &str = "state.json"; +const WAREHOUSE_INDEX_MAX_PREFIX_DEPTH: usize = 64; +const EXTERNAL_CATALOG_ROOT: &str = "external-catalog"; +const EXTERNAL_CATALOG_BRIDGE_FILE: &str = "bridge.json"; +const MAINTENANCE_ROOT: &str = "maintenance"; +const MAINTENANCE_CONFIG_FILE: &str = "config.json"; +const MAINTENANCE_JOB_ROOT: &str = "jobs"; +const MAINTENANCE_LATEST_JOB_FILE: &str = "latest.json"; +const MAINTENANCE_CURRENT_JOB_FILE: &str = "current.json"; +const MAINTENANCE_JOB_ALIAS_LATEST: &str = "latest"; +const MAINTENANCE_JOB_ALIAS_CURRENT: &str = "current"; +const TABLE_CATALOG_LIST_MAX_KEYS: usize = 1000; +const OBJECT_CATALOG_LIST_CURSOR_PREFIX: &str = "object:"; +const STRONG_CATALOG_LIST_CURSOR_PREFIX: &str = "strong:"; +const TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS: i64 = 15 * 60; +const TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS: u64 = 24 * 60 * 60; +const TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_DEFAULT_SECONDS: u64 = 15 * 60; +const TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_MAX_SECONDS: u64 = 24 * 60 * 60; +const TABLE_MAINTENANCE_SCHEDULER_AUDIT_LIMIT: usize = 10; +const TABLE_MAINTENANCE_DELETE_DISABLED_REASON: &str = "metadata delete is disabled by maintenance config"; +const TABLE_COMMIT_SLOW_LOG_THRESHOLD: StdDuration = StdDuration::from_secs(2); +const ICEBERG_MAIN_REF: &str = "main"; +const ICEBERG_MIN_SNAPSHOTS_TO_KEEP_PROPERTY: &str = "history.expire.min-snapshots-to-keep"; +const ICEBERG_MAX_SNAPSHOT_AGE_MS_PROPERTY: &str = "history.expire.max-snapshot-age-ms"; +const ICEBERG_MAX_REF_AGE_MS_PROPERTY: &str = "history.expire.max-ref-age-ms"; +const ICEBERG_REF_MIN_SNAPSHOTS_TO_KEEP_FIELD: &str = "min-snapshots-to-keep"; +const ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD: &str = "max-snapshot-age-ms"; +const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms"; +const STRONG_TABLE_CATALOG_SNAPSHOT_VERSION: u16 = 1; +const STRONG_TABLE_CATALOG_BACKING_ROOT: &str = "strong-backing"; +const STRONG_TABLE_CATALOG_SNAPSHOT_FILE: &str = "snapshot.json"; +const TABLE_CATALOG_MIGRATION_VERSION: u16 = 1; +const TABLE_CATALOG_MIGRATION_ROOT: &str = "backing-migration"; +const TABLE_CATALOG_MIGRATION_FENCE_FILE: &str = "durable-strong-fence.json"; +const TABLE_CATALOG_MIGRATION_FENCE_LOCK: &str = "durable-strong-fence.lock"; +const TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_FILE: &str = "durable-strong-global-fence.json"; +const TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_LOCK: &str = "durable-strong-global-fence.lock"; + +type CatalogListObjectsV2Info = StorageListObjectsV2Info; +type CatalogListObjectVersionsInfo = StorageListObjectVersionsInfo; +type CatalogObjectInfoOrErr = StorageObjectInfoOrErr; +type CatalogWalkOptions = StorageWalkOptions bool>; + +pub(crate) trait TableCatalogStorage: + StorageObjectIO< + Error = EcstoreError, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + StorageObjectOperations< + Error = EcstoreError, + ObjectInfo = ObjectInfo, + ObjectOptions = ObjectOptions, + FileInfo = FileInfo, + ObjectToDelete = ObjectToDelete, + DeletedObject = DeletedObject, + > + StorageListOperations< + Error = EcstoreError, + ListObjectsV2Info = CatalogListObjectsV2Info, + ListObjectVersionsInfo = CatalogListObjectVersionsInfo, + ObjectInfoOrErr = CatalogObjectInfoOrErr, + WalkOptions = CatalogWalkOptions, + WalkCancellation = tokio_util::sync::CancellationToken, + WalkResultSender = tokio::sync::mpsc::Sender, + > + StorageNamespaceLocking +{ +} + +impl TableCatalogStorage for T where + T: StorageObjectIO< + Error = EcstoreError, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + StorageObjectOperations< + Error = EcstoreError, + ObjectInfo = ObjectInfo, + ObjectOptions = ObjectOptions, + FileInfo = FileInfo, + ObjectToDelete = ObjectToDelete, + DeletedObject = DeletedObject, + > + StorageListOperations< + Error = EcstoreError, + ListObjectsV2Info = CatalogListObjectsV2Info, + ListObjectVersionsInfo = CatalogListObjectVersionsInfo, + ObjectInfoOrErr = CatalogObjectInfoOrErr, + WalkOptions = CatalogWalkOptions, + WalkCancellation = tokio_util::sync::CancellationToken, + WalkResultSender = tokio::sync::mpsc::Sender, + > + StorageNamespaceLocking +{ +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableCatalogBackingMode { + ObjectBacked, + DurableStrong, +} + +impl TableCatalogBackingMode { + pub(crate) fn from_env() -> TableCatalogStoreResult { + match std::env::var(ENV_TABLE_CATALOG_BACKING) { + Ok(value) => Self::parse(&value), + Err(std::env::VarError::NotPresent) => Ok(Self::ObjectBacked), + Err(std::env::VarError::NotUnicode(_)) => Err(TableCatalogStoreError::Invalid(format!( + "{ENV_TABLE_CATALOG_BACKING} must be valid UTF-8" + ))), + } + } + + fn parse(value: &str) -> TableCatalogStoreResult { + match value.trim() { + "" | TABLE_CATALOG_BACKING_OBJECT => Ok(Self::ObjectBacked), + TABLE_CATALOG_BACKING_DURABLE_STRONG => Ok(Self::DurableStrong), + value => Err(TableCatalogStoreError::Invalid(format!( + "unsupported table catalog backing {value}; expected {TABLE_CATALOG_BACKING_OBJECT} or {TABLE_CATALOG_BACKING_DURABLE_STRONG}" + ))), + } + } + + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::ObjectBacked => TABLE_CATALOG_BACKING_OBJECT, + Self::DurableStrong => TABLE_CATALOG_BACKING_DURABLE_STRONG, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TableCatalogListPage { + pub entries: Vec, + pub next_cursor: Option, +} + +fn finish_catalog_list_page( + mut entries: Vec, + limit: NonZeroUsize, + cursor_prefix: &str, + key: F, +) -> TableCatalogListPage +where + F: Fn(&T) -> &str, +{ + let next_cursor = if entries.len() > limit.get() { + entries.truncate(limit.get()); + entries.last().map(|entry| format!("{cursor_prefix}{}", key(entry))) + } else { + None + }; + TableCatalogListPage { entries, next_cursor } +} + +fn catalog_list_page_from_entries( + mut entries: Vec, + cursor: Option<&str>, + limit: NonZeroUsize, + key: F, +) -> TableCatalogListPage +where + F: Fn(&T) -> &str, +{ + entries.sort_by(|left, right| key(left).cmp(key(right))); + let start = cursor.map_or(0, |cursor| entries.partition_point(|entry| key(entry) <= cursor)); + let entries = entries.into_iter().skip(start).take(limit.get().saturating_add(1)).collect(); + finish_catalog_list_page(entries, limit, "", key) +} + +fn catalog_list_cursor<'a>(cursor: Option<&'a str>, prefix: &str) -> TableCatalogStoreResult> { + cursor + .map(|cursor| { + cursor + .strip_prefix(prefix) + .filter(|cursor| !cursor.is_empty()) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("page cursor does not match the active table catalog backing".to_string()) + }) + }) + .transpose() +} + +fn parse_namespace_for_store(namespace: &str) -> TableCatalogStoreResult { + Namespace::parse(namespace).map_err(|err| TableCatalogStoreError::Invalid(format!("invalid namespace: {err}"))) +} + +fn parse_table_for_store(table: &str) -> TableCatalogStoreResult { + IdentifierSegment::parse(table).map_err(|err| TableCatalogStoreError::Invalid(format!("invalid table name: {err}"))) +} + +fn http_preconditions_for_catalog_put(precondition: TableCatalogPutPrecondition) -> Option { + match precondition { + TableCatalogPutPrecondition::Any => None, + TableCatalogPutPrecondition::IfAbsent => Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + TableCatalogPutPrecondition::IfMatch(etag) => Some(HTTPPreconditions { + if_match: Some(etag), + ..Default::default() + }), + } +} + +fn is_missing_storage_error(err: &StorageError) -> bool { + matches!( + err, + StorageError::ObjectNotFound(_, _) | StorageError::FileNotFound | StorageError::ConfigNotFound + ) +} + +fn storage_error_to_catalog(action: &str, err: StorageError) -> TableCatalogStoreError { + match err { + StorageError::ObjectNotFound(bucket, object) => TableCatalogStoreError::NotFound(format!("{action}: {bucket}/{object}")), + StorageError::BucketNotFound(bucket) => TableCatalogStoreError::NotFound(format!("{action}: bucket {bucket}")), + StorageError::PreconditionFailed => TableCatalogStoreError::Conflict(format!("{action}: precondition failed")), + other => TableCatalogStoreError::Internal(format!("{action}: {other}")), + } +} + +#[cfg(test)] +mod tests; diff --git a/rustfs/src/table_catalog/model.rs b/rustfs/src/table_catalog/model.rs new file mode 100644 index 000000000..efc13e685 --- /dev/null +++ b/rustfs/src/table_catalog/model.rs @@ -0,0 +1,1353 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableBucketMarker { + pub version: u16, + pub catalog_type: &'static str, + pub reserved_prefix: &'static str, +} + +impl Default for TableBucketMarker { + fn default() -> Self { + Self { + version: TABLE_BUCKET_CONFIG_VERSION, + catalog_type: TABLE_BUCKET_CATALOG_TYPE, + reserved_prefix: TABLE_RESERVED_PREFIX, + } + } +} + +pub(crate) fn table_bucket_marker_json() -> Result, serde_json::Error> { + serde_json::to_vec(&TableBucketMarker::default()) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogEntryState { + Active, + Deleting, + Deleted, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableBucketEntry { + pub version: u16, + pub table_bucket: String, + pub catalog_type: String, + pub warehouse_root: String, + pub state: TableCatalogEntryState, + #[serde(default)] + pub properties: BTreeMap, + pub created_at: Option, + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct NamespaceEntry { + pub version: u16, + pub table_bucket: String, + pub namespace: String, + pub namespace_id: String, + pub state: TableCatalogEntryState, + #[serde(default)] + pub properties: BTreeMap, + pub created_at: Option, + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableEntry { + pub version: u16, + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub table_id: String, + pub table_uuid: String, + pub format: String, + pub format_version: u16, + pub warehouse_location: String, + pub metadata_location: String, + pub version_token: String, + pub generation: u64, + pub state: TableCatalogEntryState, + #[serde(default)] + pub properties: BTreeMap, + pub created_at: Option, + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableWarehouseIndexEntry { + pub(super) version: u16, + pub(super) table_bucket: String, + pub(super) namespace: String, + pub(super) table: String, + pub(super) table_id: String, + pub(super) warehouse_object_prefix: String, + pub(super) state: TableCatalogEntryState, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableWarehouseIndexStateEntry { + pub(super) version: u16, + pub(super) table_bucket: String, + pub(super) state: TableCatalogEntryState, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum WarehouseIndexReservation { + Created, + AlreadyReserved, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ViewEntry { + pub version: u16, + pub table_bucket: String, + pub namespace: String, + pub view: String, + pub view_id: String, + pub view_uuid: String, + pub format: String, + pub format_version: u16, + pub warehouse_location: String, + pub metadata_location: String, + pub version_token: String, + pub generation: u64, + pub state: TableCatalogEntryState, + #[serde(default)] + pub properties: BTreeMap, + pub created_at: Option, + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum CommitLogStatus { + Staged, + Committed, + Failed, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct CommitLogEntry { + pub version: u16, + pub commit_id: String, + pub idempotency_key: Option, + pub table_id: String, + pub operation: String, + pub expected_version_token: String, + pub new_version_token: String, + pub previous_metadata_location: String, + pub new_metadata_location: String, + #[serde(default)] + pub requirements: Vec, + pub status: CommitLogStatus, + pub writer: Option, + pub created_at: Option, + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableCommitRequest { + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub commit_id: String, + pub idempotency_key: Option, + pub operation: String, + pub expected_version_token: String, + pub expected_metadata_location: String, + pub new_metadata_location: String, + #[serde(default)] + pub requirements: Vec, + pub writer: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableCommitResult { + pub table: TableEntry, + pub commit_log: CommitLogEntry, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ExternalCatalogBridgeEntry { + pub version: u16, + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub catalog: String, + pub external_catalog_id: Option, + pub external_namespace: String, + pub external_table: String, + pub external_table_uuid: Option, + pub metadata_location: Option, + pub external_version_token: Option, + pub policy_mode: String, + pub credential_mode: String, + pub sync_mode: String, + pub rollback_strategy: String, + pub last_sync_status: Option, + pub last_synced_metadata_location: Option, + #[serde(default)] + pub properties: BTreeMap, + pub created_at: Option, + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ViewCommitRequest { + pub table_bucket: String, + pub namespace: String, + pub view: String, + pub expected_version_token: String, + pub expected_metadata_location: String, + pub new_metadata_location: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ViewCommitResult { + pub view: ViewEntry, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableMaintenanceConfig { + pub version: u16, + #[serde(rename = "retain-recent-metadata-files")] + pub retain_recent_metadata_files: usize, + #[serde(rename = "delete-enabled")] + pub delete_enabled: bool, + #[serde(rename = "background-enabled")] + pub background_enabled: bool, + #[serde(default, rename = "worker-paused")] + pub worker_paused: bool, + #[serde( + default = "default_table_maintenance_worker_lease_timeout_seconds", + rename = "worker-lease-timeout-seconds" + )] + pub worker_lease_timeout_seconds: u64, + #[serde(default, rename = "max-retry-attempts")] + pub max_retry_attempts: u16, + #[serde(default, rename = "retry-initial-backoff-seconds")] + pub retry_initial_backoff_seconds: u64, + #[serde(default, rename = "retry-max-backoff-seconds")] + pub retry_max_backoff_seconds: u64, + #[serde(default, rename = "quarantine-enabled")] + pub quarantine_enabled: bool, + #[serde(default, rename = "quarantine-retention-seconds")] + pub quarantine_retention_seconds: u64, +} + +impl Default for TableMaintenanceConfig { + fn default() -> Self { + Self { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 0, + delete_enabled: false, + background_enabled: false, + worker_paused: false, + worker_lease_timeout_seconds: TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_DEFAULT_SECONDS, + max_retry_attempts: 0, + retry_initial_backoff_seconds: 5, + retry_max_backoff_seconds: 300, + quarantine_enabled: false, + quarantine_retention_seconds: 0, + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMaintenanceConfigSource { + #[default] + Default, + TableBucketDefault, + TableOverride, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableMaintenanceEffectiveConfig { + pub config: TableMaintenanceConfig, + pub source: TableMaintenanceConfigSource, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMaintenanceSchedulerStatus { + Ready, + Queued, + Disabled, + Paused, + Backpressured, + RetryDeferred, + Quarantined, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMaintenanceSchedulerReport { + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub table_id: String, + pub status: TableMaintenanceSchedulerStatus, + pub config_source: TableMaintenanceConfigSource, + pub background_enabled: bool, + pub worker_paused: bool, + pub delete_enabled: bool, + pub worker_lease_timeout_seconds: u64, + pub max_retry_attempts: u16, + pub retry_initial_backoff_seconds: u64, + pub retry_max_backoff_seconds: u64, + pub recommended_actions: Vec, + pub current_job: Option, + pub quarantine: TableMaintenanceSchedulerQuarantineBoundary, + pub audit_timeline: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMaintenanceSchedulerRunResult { + pub report: TableMetadataMaintenanceReport, + pub scheduler: TableMaintenanceSchedulerReport, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMaintenanceSchedulerQuarantineBoundary { + pub enabled: bool, + pub active: bool, + pub retention_seconds: u64, + pub quarantined_object_count: usize, + pub source_job_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMaintenanceAuditActor { + Scheduler, + Worker, + Operator, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMaintenanceAuditAction { + Planned, + WorkerControl, + SchedulerControl, + SchedulerQueued, + SchedulerLeaseExpired, + WorkerStarted, + WorkerHeartbeat, + WorkerLeaseExpired, + WorkerSucceeded, + WorkerFailed, + QuarantineRelease, + QuarantineRetry, + QuarantineAbandon, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMaintenanceAuditEvent { + pub timestamp: String, + pub actor: TableMaintenanceAuditActor, + pub action: TableMaintenanceAuditAction, + #[serde(default)] + pub reason: Option, + #[serde(default, rename = "before-status")] + pub before_status: Option, + #[serde(default, rename = "after-status")] + pub after_status: Option, + #[serde(default, rename = "before-quarantined-object-count")] + pub before_quarantined_object_count: Option, + #[serde(default, rename = "after-quarantined-object-count")] + pub after_quarantined_object_count: Option, + #[serde(default, rename = "recommended-actions")] + pub recommended_actions: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMaintenanceQuarantineAction { + Inspect, + Release, + Retry, + Abandon, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableMaintenanceQuarantineOperationRequest { + pub action: TableMaintenanceQuarantineAction, + #[serde(default)] + pub reason: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMaintenanceQuarantineOperationResult { + pub action: TableMaintenanceQuarantineAction, + pub report: TableMetadataMaintenanceReport, + pub scheduler: TableMaintenanceSchedulerReport, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMaintenanceSchedulerJobSummary { + pub job_id: String, + pub operation: TableMetadataMaintenanceOperation, + pub status: TableMetadataMaintenanceJobStatus, + #[serde(default, rename = "scheduler-id")] + pub scheduler_id: Option, + #[serde(default, rename = "scheduled-at")] + pub scheduled_at: Option, + pub worker_id: Option, + pub attempt: u16, + pub started_at: Option, + pub finished_at: Option, + pub heartbeat_at: Option, + pub next_retry_after: Option, + pub recommended_actions: Vec, + #[serde(default, rename = "audit-events")] + pub audit_events: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMetadataMaintenanceJob { + pub job_id: String, + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub table_id: String, + #[serde(default)] + pub operation: TableMetadataMaintenanceOperation, + #[serde(default)] + pub status: TableMetadataMaintenanceJobStatus, + #[serde(default)] + pub failure_reason: Option, + #[serde(default, rename = "recommended-actions")] + pub recommended_actions: Vec, + #[serde(default)] + pub config_source: TableMaintenanceConfigSource, + #[serde(default, rename = "scheduler-id")] + pub scheduler_id: Option, + #[serde(default, rename = "scheduler-lease-id")] + pub scheduler_lease_id: String, + #[serde(default, rename = "scheduled-at")] + pub scheduled_at: Option, + #[serde(default)] + pub worker_id: Option, + #[serde(default)] + pub lease_id: String, + #[serde(default)] + pub attempt: u16, + #[serde(default, rename = "max-retry-attempts")] + pub max_retry_attempts: u16, + #[serde(default, rename = "next-retry-after")] + pub next_retry_after: Option, + #[serde(default, rename = "quarantine-enabled")] + pub quarantine_enabled: bool, + #[serde(default, rename = "quarantine-retention-seconds")] + pub quarantine_retention_seconds: u64, + #[serde(default)] + pub heartbeat_at: Option, + #[serde(default)] + pub started_at: Option, + #[serde(default)] + pub finished_at: Option, + pub current_metadata_location: String, + pub current_generation: u64, + pub retain_recent_metadata_files: usize, + pub safety_window_seconds: i64, + pub cleanup_watermark_unix_seconds: i64, + #[serde(default)] + pub planned_metadata_file_count: usize, + #[serde(default)] + pub retained_metadata_file_count: usize, + #[serde(default)] + pub cleanup_candidate_count: usize, + #[serde(default)] + pub deletable_metadata_file_count: usize, + #[serde(default)] + pub deleted_metadata_file_count: usize, + #[serde(default)] + pub planned_object_file_count: usize, + #[serde(default)] + pub cleanup_candidate_object_count: usize, + #[serde(default)] + pub deletable_object_count: usize, + #[serde(default)] + pub deleted_object_count: usize, + #[serde(default)] + pub quarantined_object_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMetadataMaintenanceReport { + pub job: TableMetadataMaintenanceJob, + pub current_metadata_location: String, + pub retained_metadata_locations: Vec, + pub cleanup_candidate_locations: Vec, + pub deletable_metadata_locations: Vec, + #[serde(default, rename = "cleanup-object-candidate-locations")] + pub cleanup_object_candidate_locations: Vec, + #[serde(default, rename = "deletable-object-locations")] + pub deletable_object_locations: Vec, + #[serde(default)] + pub object_reports: Vec, + #[serde(default, rename = "object-cleanup-reports")] + pub object_cleanup_reports: Vec, + #[serde(default)] + pub referenced_object_reports: Vec, + #[serde(default, rename = "reachability-graph")] + pub reachability_graph: TableMaintenanceReachabilityGraphReport, + #[serde(default, rename = "snapshot-expiration")] + pub snapshot_expiration: Option, + #[serde(default)] + pub compaction: Option, + #[serde(default, rename = "audit-events")] + pub audit_events: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMaintenanceReachabilityGraphReport { + pub status: TableMaintenanceReachabilityGraphStatus, + pub metadata_file_count: usize, + pub manifest_list_count: usize, + pub manifest_file_count: usize, + pub data_file_count: usize, + pub delete_file_count: usize, + pub manual_review_count: usize, + pub reasons: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMaintenanceReachabilityGraphStatus { + Complete, + #[default] + ManualReviewRequired, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMaintenanceReachabilityGraphReason { + MetadataJsonParsed, + ManifestListAvroReferenced, + ManifestAvroReaderUnavailable, + DataFileCleanupDeferred, + DeleteFileCleanupDeferred, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableSnapshotExpirationConfig { + #[serde(rename = "min-snapshots-to-keep")] + pub min_snapshots_to_keep: usize, + #[serde(rename = "max-snapshot-age-ms")] + pub max_snapshot_age_ms: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableSnapshotExpirationReport { + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub table_id: String, + pub current_metadata_location: String, + pub current_snapshot_id: Option, + pub config: TableSnapshotExpirationConfig, + pub expiration_watermark_ms: i64, + pub retained_snapshot_count: usize, + pub expiration_candidate_count: usize, + pub manual_review_count: usize, + #[serde(default)] + pub expired_snapshot_ids: Vec, + #[serde(default)] + pub committed_metadata_location: Option, + pub snapshot_reports: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableSnapshotExpirationSnapshotReport { + pub snapshot_id: Option, + pub sequence_number: Option, + pub timestamp_ms: Option, + pub manifest_list: Option, + pub state: TableSnapshotExpirationSnapshotState, + pub reasons: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableSnapshotExpirationSnapshotState { + Retained, + ExpirationCandidate, + ManualReviewRequired, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableSnapshotExpirationReason { + CurrentSnapshot, + MinSnapshotsToKeep, + ProtectedSnapshotRef, + UserDefinedSnapshotRef, + SnapshotRefRetentionConflict, + TableRetentionPropertyConflict, + MissingSnapshotId, + MissingSnapshotTimestamp, + SnapshotAgeWithinRetention, + SnapshotAgeExpired, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableCompactionPlanningConfig { + #[serde(rename = "target-file-size-bytes")] + pub target_file_size_bytes: u64, + #[serde(rename = "small-file-threshold-bytes")] + pub small_file_threshold_bytes: u64, + #[serde(rename = "min-input-files")] + pub min_input_files: usize, + #[serde(rename = "max-rewrite-bytes-per-job")] + pub max_rewrite_bytes_per_job: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableCompactionPlanningReport { + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub table_id: String, + pub current_metadata_location: String, + pub current_snapshot_id: Option, + pub config: TableCompactionPlanningConfig, + pub status: TableCompactionPlanningStatus, + pub candidate_file_count: usize, + pub rewrite_group_count: usize, + pub manual_review_count: usize, + #[serde(default, rename = "committed-metadata-location")] + pub committed_metadata_location: Option, + #[serde(default, rename = "row-level-planning")] + pub row_level_planning: TableRowLevelMaintenancePlanningReport, + #[serde(default, rename = "rewrite-groups")] + pub rewrite_groups: Vec, + pub snapshot_reports: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableRowLevelMaintenancePlanningReport { + pub status: TableRowLevelMaintenancePlanningStatus, + #[serde(rename = "delete-file-count")] + pub delete_file_count: usize, + #[serde(rename = "position-delete-file-count")] + pub position_delete_file_count: usize, + #[serde(rename = "equality-delete-file-count")] + pub equality_delete_file_count: usize, + #[serde(rename = "manual-review-count")] + pub manual_review_count: usize, + pub reasons: Vec, + #[serde(rename = "delete-files")] + pub delete_files: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableRowLevelMaintenancePlanningStatus { + #[default] + NoDeleteFiles, + ManualReviewRequired, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableRowLevelMaintenancePlanningReason { + PositionDeleteFile, + EqualityDeleteFile, + DeleteFileRewriteUnsupported, + MissingDeleteFile, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableRowLevelDeleteFilePlanningReport { + #[serde(rename = "file-location")] + pub file_location: String, + pub content: TableRowLevelDeleteFileContent, + #[serde(rename = "object-exists")] + pub object_exists: bool, + #[serde(default, rename = "record-count", skip_serializing_if = "Option::is_none")] + pub record_count: Option, + #[serde(default, rename = "file-size-bytes", skip_serializing_if = "Option::is_none")] + pub file_size_bytes: Option, + #[serde(default, rename = "sequence-number", skip_serializing_if = "Option::is_none")] + pub sequence_number: Option, + #[serde(default, rename = "file-sequence-number", skip_serializing_if = "Option::is_none")] + pub file_sequence_number: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableRowLevelDeleteFileContent { + PositionDelete, + EqualityDelete, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableCompactionRewriteGroup { + pub group_id: String, + #[serde(default, rename = "sort-order-id", skip_serializing_if = "Option::is_none")] + pub sort_order_id: Option, + #[serde(rename = "input-file-locations")] + pub input_file_locations: Vec, + #[serde(rename = "input-file-count")] + pub input_file_count: usize, + #[serde(rename = "input-bytes")] + pub input_bytes: u64, + #[serde(default, rename = "output-file-location")] + pub output_file_location: Option, + #[serde(default, rename = "output-bytes")] + pub output_bytes: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableCompactionSnapshotReport { + pub snapshot_id: Option, + pub manifest_list: Option, + pub status: TableCompactionPlanningStatus, + pub reasons: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCompactionPlanningStatus { + NoCandidates, + RewriteCandidates, + Committed, + ManualReviewRequired, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCompactionPlanningReason { + ManifestList, + ManifestFile, + SmallDataFile, + RewriteGroup, + CompactionCommitted, + ManifestAvroReaderUnavailable, + MissingCurrentSnapshot, + MissingManifestList, + MissingDataFile, + DeleteFile, + PositionDeleteFile, + EqualityDeleteFile, + RowLevelRewriteUnsupported, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMetadataMaintenanceOperation { + #[default] + DryRun, + Delete, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMetadataMaintenanceJobStatus { + NotYetRun, + Queued, + Running, + #[default] + Successful, + Failed, + Disabled, + Paused, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMaintenanceRecommendedAction { + NoActionRequired, + RunMaintenanceWorker, + ReviewAndRunDelete, + ReviewQuarantine, + EnableDelete, + EnableBackgroundMaintenance, + ResumeMaintenanceWorker, + WaitForRetryBackoff, + WaitForActiveWorker, + InvestigateFailure, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMetadataMaintenanceObjectState { + Retained, + PendingSafetyWindow, + Deletable, + Deleted, + ManualReviewRequired, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMetadataMaintenanceReason { + CurrentMetadata, + MetadataLog, + ProtectedSnapshotRef, + RecentMetadata, + NoCurrentReachability, + SafetyWindowPending, + SafetyWindowSatisfied, + DeletedByMaintenance, + ManifestList, + ManifestFile, + DataFile, + DeleteFile, + UnsupportedManifestAvro, + UnreadableMetadata, + QuarantineEnabled, + RetryScheduled, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMetadataMaintenanceObjectKind { + MetadataFile, + ManifestList, + ManifestFile, + DataFile, + DeleteFile, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMetadataMaintenanceObjectReport { + pub metadata_location: String, + pub state: TableMetadataMaintenanceObjectState, + pub reasons: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMetadataMaintenanceObjectCleanupReport { + pub object_location: String, + pub object_kind: TableMetadataMaintenanceObjectKind, + pub state: TableMetadataMaintenanceObjectState, + pub reasons: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMetadataMaintenanceReferencedObjectReport { + pub object_location: String, + pub object_kind: TableMetadataMaintenanceObjectKind, + pub state: TableMetadataMaintenanceObjectState, + pub reasons: Vec, +} + +pub(crate) struct TableMaintenanceHeartbeatRef<'a> { + pub(super) table_bucket: &'a str, + pub(super) namespace: &'a str, + pub(super) table: &'a str, + pub(super) job_id: &'a str, + pub(super) lease_id: &'a str, + pub(super) worker_id: &'a str, +} + +pub(crate) struct TableMaintenancePreflightContext<'a> { + pub(super) table_bucket: &'a str, + pub(super) namespace: &'a Namespace, + pub(super) table: &'a IdentifierSegment, + pub(super) entry: &'a TableEntry, +} + +pub(crate) struct TableMaintenanceWorkerControlReport<'a> { + pub(super) table_bucket: &'a str, + pub(super) namespace: &'a Namespace, + pub(super) table: &'a IdentifierSegment, + pub(super) entry: &'a TableEntry, + pub(super) worker_id: String, + pub(super) effective: &'a TableMaintenanceEffectiveConfig, + pub(super) status: TableMetadataMaintenanceJobStatus, + pub(super) reason: &'a str, + pub(super) now: OffsetDateTime, +} + +pub(crate) struct TableMaintenanceSchedulerControlReport<'a> { + pub(super) table_bucket: &'a str, + pub(super) namespace: &'a Namespace, + pub(super) table: &'a IdentifierSegment, + pub(super) entry: &'a TableEntry, + pub(super) scheduler_id: String, + pub(super) effective: &'a TableMaintenanceEffectiveConfig, + pub(super) status: TableMetadataMaintenanceJobStatus, + pub(super) reason: &'a str, + pub(super) now: OffsetDateTime, +} + +pub(crate) enum TableMaintenanceWorkerPreflight { + Ready { + effective: TableMaintenanceEffectiveConfig, + queued: Option>, + }, + Complete(Box), +} + +pub(crate) enum TableMaintenanceSchedulerPreflight { + Ready(TableMaintenanceEffectiveConfig), + Complete(Box), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogExport { + pub table_bucket: TableBucketEntry, + pub namespace: NamespaceEntry, + pub table: TableEntry, + pub backing_manifest: TableCatalogBackingManifest, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogBackingManifest { + pub version: u16, + pub current: TableCatalogBackingProfile, + pub migration: TableCatalogBackingMigrationPlan, + pub ha: TableCatalogHaPolicy, + pub scale_validation: TableCatalogScaleValidation, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogBackingProfile { + pub kind: TableCatalogBackingKind, + pub authority: TableCatalogAuthority, + pub consistency: TableCatalogConsistencyMode, + pub durability: TableCatalogDurabilityMode, + pub current_pointer_path: String, + pub wal: TableCatalogWalState, + pub snapshot: TableCatalogSnapshotState, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogBackingKind { + ObjectBacked, + StrongKvWal, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogAuthority { + RustfsSysObject, + LinearizableMetadataKv, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogConsistencyMode { + ConditionalObjectCas, + LinearizableCas, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogDurabilityMode { + StagedCommitLogBeforePointerUpdate, + WalBeforeStateMachineApply, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogWalState { + pub status: TableCatalogWalStatus, + pub commit_log_prefix: String, + pub idempotency_index_prefix: String, + pub committed_generation: u64, + pub staged_before_table_update_count: usize, + pub finalization_required_count: usize, + pub idempotency_repair_required_count: usize, + pub manual_review_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogWalStatus { + Recoverable, + RecoveryRequired, + ManualReviewRequired, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogSnapshotState { + pub export_api: String, + pub includes_table_bucket: bool, + pub includes_namespace: bool, + pub includes_table_pointer: bool, + pub includes_backing_manifest: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogBackingMigrationPlan { + pub source_kind: TableCatalogBackingKind, + pub target_kind: TableCatalogBackingKind, + pub status: TableCatalogBackingMigrationStatus, + pub required_steps: Vec, + pub blockers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogBackingMigrationDryRunReport { + pub table_bucket: String, + pub source_kind: TableCatalogBackingKind, + pub target_kind: TableCatalogBackingKind, + pub status: TableCatalogBackingMigrationStatus, + pub namespace_count: usize, + pub table_count: usize, + pub view_count: usize, + pub commit_log_count: usize, + pub idempotency_index_count: usize, + pub warehouse_prefix_count: usize, + pub warehouse_index_ready: bool, + pub object_backed_writes_fenced: bool, + pub ready_to_enable_durable_strong: bool, + pub blockers: Vec, + pub recommended_actions: Vec, + pub rollback: TableCatalogBackingRollbackPlan, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogBackingMigrationExecutionReport { + pub table_bucket: String, + pub source_kind: TableCatalogBackingKind, + pub target_kind: TableCatalogBackingKind, + pub status: TableCatalogBackingMigrationExecutionStatus, + pub namespace_count: usize, + pub table_count: usize, + pub view_count: usize, + pub commit_log_count: usize, + pub idempotency_index_count: usize, + pub source_fingerprint: String, + pub target_snapshot_etag: String, + pub object_backed_writes_fenced: bool, + pub ready_to_enable_durable_strong: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogBackingMigrationExecutionStatus { + SnapshotMaterialized, + SnapshotAlreadyMaterialized, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogBackingMigrationCancelReport { + pub table_bucket: String, + pub status: TableCatalogBackingMigrationCancelStatus, + pub object_backed_writes_fenced: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogBackingMigrationCancelStatus { + FenceReleased, + NoMigrationFence, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogBackingMigrationStatus { + ReadyToSnapshot, + SnapshotMaterialized, + RecoveryRequired, + ManualReviewRequired, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogBackingMigrationStep { + SnapshotCatalogExport, + ReplayCommitLog, + VerifyCurrentPointer, + EnableSingleWriterFencing, + CutOverLinearizableReads, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogBackingMigrationBlocker { + CommitRecoveryRequired, + CommitManualReviewRequired, + WarehouseIndexBackfillRequired, + DuplicateWarehousePrefix, + DurableStrongSnapshotChanged, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogBackingMigrationAction { + RunCatalogRecovery, + BackfillWarehouseIndex, + ReviewDuplicateWarehousePrefixes, + SnapshotObjectBackedCatalog, + EnableDurableStrongBacking, + VerifyDurableStrongSnapshot, + SnapshotRemainingTableBuckets, + ReviewDurableStrongSnapshot, + KeepObjectBackedRollbackConfig, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogBackingRollbackPlan { + pub backing_config_key: &'static str, + pub current_backing_value: &'static str, + pub rollback_backing_value: &'static str, + pub preserves_object_backed_catalog: bool, + pub requires_operator_restart: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogHaPolicy { + pub writer_region_model: TableCatalogHaWriterModel, + pub read_replica_strategy: TableCatalogReadReplicaStrategy, + pub commit_read_requirement: TableCatalogCommitReadRequirement, + pub active_active_supported: bool, + pub failover_requires_operator_promotion: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogHaWriterModel { + SingleActiveWriterRegion, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogReadReplicaStrategy { + ReadOnlyReplicasForListAndLoad, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogCommitReadRequirement { + LinearizableLeaderRead, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogScaleValidation { + pub status: TableCatalogScaleValidationStatus, + pub benchmark_required: bool, + pub required_scenarios: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogScaleValidationStatus { + MatrixPublished, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogScaleValidationScenario { + ConcurrentCommitCas, + CommitLogRecoveryReplay, + MigrationSnapshotReplay, + ReadReplicaStaleReadGuard, + ClientConformanceMatrix, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TableDataPlaneResource { + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub table_id: String, + pub warehouse_object_prefix: String, +} + +impl TableDataPlaneResource { + pub(crate) fn catalog_resource_object(&self) -> String { + let namespace = Namespace::parse(&self.namespace) + .map(|namespace| namespace.storage_id()) + .unwrap_or_else(|_| self.namespace.clone()); + format!("{NAMESPACE_ROOT}/{namespace}/{TABLE_ROOT}/{}", self.table) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableMetadataPointerStatus { + Valid, + MissingObject, + InvalidLocation, + InvalidJson, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogRecoveryStatus { + Healthy, + Recoverable, + ManualReviewRequired, + ReadOnlyRecommended, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogRecoveryAction { + RunCommitRecovery, + RetryCommit, + RestoreCurrentMetadataObject, + FixCurrentMetadataJson, + MoveCurrentMetadataInsideTable, + ReviewCommitLog, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogDiagnosticsReport { + pub catalog: TableCatalogExport, + pub current_metadata_status: TableMetadataPointerStatus, + pub recovery_status: TableCatalogRecoveryStatus, + pub recommended_actions: Vec, + pub commit_recovery: TableCommitRecoveryReport, + pub backing_manifest: TableCatalogBackingManifest, + pub orphan_metadata_candidate_locations: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCommitRecoveryState { + Committed, + StagedBeforeTableUpdate, + FinalizationRequired, + IdempotencyIndexRepairRequired, + ManualReview, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCommitIdempotencyIndexStatus { + NotRequired, + Missing, + Matches, + Stale, + Conflicting, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCommitRecoveryEntry { + pub commit_id: String, + pub idempotency_key: Option, + pub operation: String, + pub status: CommitLogStatus, + pub recovery_state: TableCommitRecoveryState, + pub previous_metadata_location: String, + pub new_metadata_location: String, + pub expected_version_token: String, + pub new_version_token: String, + pub idempotency_index_present: bool, + pub idempotency_index_status: TableCommitIdempotencyIndexStatus, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCommitRecoveryReport { + pub table_bucket: String, + pub namespace: String, + pub table: String, + pub table_id: String, + pub current_metadata_location: String, + pub current_version_token: String, + pub current_generation: u64, + pub commits: Vec, + pub staged_before_table_update_count: usize, + pub finalization_required_count: usize, + pub idempotency_repair_required_count: usize, + pub manual_review_count: usize, + pub finalized_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct NamespaceMarker { + pub version: u16, + pub namespace: String, +} + +impl NamespaceMarker { + pub fn new(namespace: &Namespace) -> Self { + Self { + version: TABLE_NAMESPACE_MARKER_VERSION, + namespace: namespace.public_name(), + } + } +} + +pub(crate) fn namespace_marker_json(namespace: &Namespace) -> Result, serde_json::Error> { + serde_json::to_vec(&NamespaceMarker::new(namespace)) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableMarker { + pub version: u16, + pub namespace: String, + pub name: String, + pub metadata_location: Option, +} + +impl TableMarker { + pub fn new(namespace: &Namespace, table: &IdentifierSegment) -> Self { + Self { + version: TABLE_RESOURCE_MARKER_VERSION, + namespace: namespace.public_name(), + name: table.as_str().to_string(), + metadata_location: None, + } + } +} + +pub(crate) fn table_marker_json(namespace: &Namespace, table: &IdentifierSegment) -> Result, serde_json::Error> { + serde_json::to_vec(&TableMarker::new(namespace, table)) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableMetadataPointer { + pub version: u16, + pub metadata_location: String, +} + +impl TableMetadataPointer { + pub fn new(metadata_location: String) -> Self { + Self { + version: TABLE_METADATA_POINTER_VERSION, + metadata_location, + } + } +} + +pub(crate) fn table_metadata_pointer_json(metadata_location: String) -> Result, serde_json::Error> { + serde_json::to_vec(&TableMetadataPointer::new(metadata_location)) +} + +pub(crate) fn parse_table_metadata_pointer(data: &[u8]) -> Result { + serde_json::from_slice(data) +} diff --git a/rustfs/src/table_catalog/store/migration.rs b/rustfs/src/table_catalog/store/migration.rs new file mode 100644 index 000000000..c94178134 --- /dev/null +++ b/rustfs/src/table_catalog/store/migration.rs @@ -0,0 +1,881 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::object::ObjectTableCatalogStore; +use super::strong::{ + StrongCommitSnapshotRecord, StrongTableCatalogBucketSnapshot, StrongTableCatalogState, TableCatalogBackingMigrationFence, + TableCatalogBackingMigrationFenceStatus, TableCatalogBackingMigrationGlobalFence, table_catalog_bucket_snapshot_fingerprint, +}; +use super::*; + +pub(super) fn table_catalog_backing_manifest( + paths: &TableCatalogObjectPaths, + namespace: &Namespace, + table: &IdentifierSegment, + entry: &TableEntry, + commit_recovery: &TableCommitRecoveryReport, +) -> TableCatalogBackingManifest { + let recovery_required = commit_recovery.staged_before_table_update_count > 0 + || commit_recovery.finalization_required_count > 0 + || commit_recovery.idempotency_repair_required_count > 0; + let manual_review_required = commit_recovery.manual_review_count > 0; + let wal_status = if manual_review_required { + TableCatalogWalStatus::ManualReviewRequired + } else if recovery_required { + TableCatalogWalStatus::RecoveryRequired + } else { + TableCatalogWalStatus::Recoverable + }; + let migration_status = if manual_review_required { + TableCatalogBackingMigrationStatus::ManualReviewRequired + } else if recovery_required { + TableCatalogBackingMigrationStatus::RecoveryRequired + } else { + TableCatalogBackingMigrationStatus::ReadyToSnapshot + }; + let mut blockers = Vec::new(); + if recovery_required { + blockers.push(TableCatalogBackingMigrationBlocker::CommitRecoveryRequired); + } + if manual_review_required { + blockers.push(TableCatalogBackingMigrationBlocker::CommitManualReviewRequired); + } + + TableCatalogBackingManifest { + version: TABLE_CATALOG_BACKING_MANIFEST_VERSION, + current: TableCatalogBackingProfile { + kind: TableCatalogBackingKind::ObjectBacked, + authority: TableCatalogAuthority::RustfsSysObject, + consistency: TableCatalogConsistencyMode::ConditionalObjectCas, + durability: TableCatalogDurabilityMode::StagedCommitLogBeforePointerUpdate, + current_pointer_path: paths.table_entry_path(&entry.table_bucket, namespace, table), + wal: TableCatalogWalState { + status: wal_status, + commit_log_prefix: paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id), + idempotency_index_prefix: paths.commit_idempotency_entries_prefix(&entry.table_bucket, &entry.table_id), + committed_generation: entry.generation, + staged_before_table_update_count: commit_recovery.staged_before_table_update_count, + finalization_required_count: commit_recovery.finalization_required_count, + idempotency_repair_required_count: commit_recovery.idempotency_repair_required_count, + manual_review_count: commit_recovery.manual_review_count, + }, + snapshot: TableCatalogSnapshotState { + export_api: "GET /iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export".to_string(), + includes_table_bucket: true, + includes_namespace: true, + includes_table_pointer: true, + includes_backing_manifest: true, + }, + }, + migration: TableCatalogBackingMigrationPlan { + source_kind: TableCatalogBackingKind::ObjectBacked, + target_kind: TableCatalogBackingKind::StrongKvWal, + status: migration_status, + required_steps: vec![ + TableCatalogBackingMigrationStep::SnapshotCatalogExport, + TableCatalogBackingMigrationStep::ReplayCommitLog, + TableCatalogBackingMigrationStep::VerifyCurrentPointer, + TableCatalogBackingMigrationStep::EnableSingleWriterFencing, + TableCatalogBackingMigrationStep::CutOverLinearizableReads, + ], + blockers, + }, + ha: TableCatalogHaPolicy { + writer_region_model: TableCatalogHaWriterModel::SingleActiveWriterRegion, + read_replica_strategy: TableCatalogReadReplicaStrategy::ReadOnlyReplicasForListAndLoad, + commit_read_requirement: TableCatalogCommitReadRequirement::LinearizableLeaderRead, + active_active_supported: false, + failover_requires_operator_promotion: true, + }, + scale_validation: TableCatalogScaleValidation { + status: TableCatalogScaleValidationStatus::MatrixPublished, + benchmark_required: true, + required_scenarios: vec![ + TableCatalogScaleValidationScenario::ConcurrentCommitCas, + TableCatalogScaleValidationScenario::CommitLogRecoveryReplay, + TableCatalogScaleValidationScenario::MigrationSnapshotReplay, + TableCatalogScaleValidationScenario::ReadReplicaStaleReadGuard, + TableCatalogScaleValidationScenario::ClientConformanceMatrix, + ], + }, + } +} + +impl ObjectTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + async fn read_backing_migration_fence( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult)>> { + self.read_entry(self.catalog_bucket(), &self.paths.backing_migration_fence_path(table_bucket)) + .await + } + + pub(super) async fn acquire_table_bucket_registry_write_permit(&self) -> TableCatalogStoreResult> { + let fence_path = self.paths.backing_migration_global_fence_path(); + let lock_path = self.paths.backing_migration_global_fence_lock_path(); + let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; + if self + .read_entry::(self.catalog_bucket(), &fence_path) + .await? + .is_some() + { + return Err(TableCatalogStoreError::Conflict( + "table bucket registry writes are fenced while durable strong migration is in progress".to_string(), + )); + } + Ok(guard) + } + + pub(super) async fn acquire_object_backed_catalog_write_permit( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult> { + let lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); + let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; + if self.read_backing_migration_fence(table_bucket).await?.is_some() { + return Err(TableCatalogStoreError::Conflict(format!( + "object-backed catalog writes are fenced while table bucket {table_bucket} is prepared for durable strong cutover" + ))); + } + Ok(guard) + } + + async fn ensure_global_backing_migration_fence( + &self, + fence_path: &str, + ) -> TableCatalogStoreResult { + if let Some((fence, _)) = self + .read_entry::(self.catalog_bucket(), fence_path) + .await? + { + if fence.version != TABLE_CATALOG_MIGRATION_VERSION { + return Err(TableCatalogStoreError::Invalid( + "invalid durable strong global migration fence".to_string(), + )); + } + return Ok(fence); + } + let fence = TableCatalogBackingMigrationGlobalFence { + version: TABLE_CATALOG_MIGRATION_VERSION, + migration_id: Uuid::new_v4().to_string(), + }; + self.write_entry(self.catalog_bucket(), fence_path, &fence, TableCatalogPutPrecondition::IfAbsent) + .await?; + Ok(fence) + } + + async fn clear_global_backing_migration_fence_if_unused(&self, fence_path: &str) -> TableCatalogStoreResult<()> { + let bucket_objects = self + .backend + .list_objects(self.catalog_bucket(), &self.paths.table_bucket_entries_prefix()) + .await?; + if bucket_objects + .iter() + .any(|object| object.ends_with(TABLE_CATALOG_MIGRATION_FENCE_FILE)) + { + return Ok(()); + } + self.backend.delete_object(self.catalog_bucket(), fence_path).await + } + + pub(super) async fn ensure_object_backed_writes_allowed(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { + if self + .backend + .object_exists(self.catalog_bucket(), &self.paths.backing_migration_fence_path(table_bucket)) + .await? + { + return Err(TableCatalogStoreError::Conflict(format!( + "object-backed catalog writes are fenced while table bucket {table_bucket} is prepared for durable strong cutover" + ))); + } + Ok(()) + } + + async fn collect_bucket_snapshot_with_locks( + &self, + table_bucket: &str, + guards: &mut Vec>, + ) -> TableCatalogStoreResult { + let bucket_path = self.paths.table_bucket_entry_path(table_bucket); + guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?); + let Some((table_bucket_entry, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), &bucket_path) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + }; + if table_bucket_entry.table_bucket != table_bucket { + return Err(TableCatalogStoreError::Invalid(format!( + "table bucket entry does not match migration target {table_bucket}" + ))); + } + + let mut namespaces = Vec::new(); + let mut tables = Vec::new(); + let mut views = Vec::new(); + let mut commits = Vec::new(); + let mut idempotency = Vec::new(); + let namespace_objects = self + .backend + .list_objects(self.catalog_bucket(), &self.paths.namespace_entries_prefix(table_bucket)) + .await?; + let mut unmatched_table_objects = namespace_objects + .iter() + .filter(|object| object.ends_with(TABLE_ENTRY_FILE)) + .cloned() + .collect::>(); + let mut unmatched_view_objects = namespace_objects + .iter() + .filter(|object| object.ends_with(VIEW_ENTRY_FILE)) + .cloned() + .collect::>(); + for namespace_object in namespace_objects + .iter() + .filter(|object| object.ends_with(NAMESPACE_ENTRY_FILE)) + { + guards.push( + self.backend + .acquire_write_lock(self.catalog_bucket(), namespace_object) + .await?, + ); + let Some((namespace_entry, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), namespace_object) + .await? + else { + return Err(TableCatalogStoreError::Conflict(format!( + "namespace changed while preparing durable strong snapshot: {namespace_object}" + ))); + }; + if namespace_entry.table_bucket != table_bucket { + return Err(TableCatalogStoreError::Invalid(format!( + "namespace {} belongs to a different table bucket", + namespace_entry.namespace + ))); + } + let namespace = parse_namespace_for_store(&namespace_entry.namespace)?; + + let table_objects = self + .backend + .list_objects(self.catalog_bucket(), &self.paths.table_entries_prefix(table_bucket, &namespace)) + .await?; + for table_object in table_objects.iter().filter(|object| object.ends_with(TABLE_ENTRY_FILE)) { + unmatched_table_objects.remove(table_object); + guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), table_object).await?); + let Some((table_entry, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), table_object) + .await? + else { + return Err(TableCatalogStoreError::Conflict(format!( + "table changed while preparing durable strong snapshot: {table_object}" + ))); + }; + if table_entry.table_bucket != table_bucket || table_entry.namespace != namespace_entry.namespace { + return Err(TableCatalogStoreError::Invalid(format!( + "table {} does not match its catalog namespace", + table_entry.table + ))); + } + + for commit_object in self + .backend + .list_objects( + self.catalog_bucket(), + &self.paths.commit_log_entries_prefix(table_bucket, &table_entry.table_id), + ) + .await? + .into_iter() + .filter(|object| object.ends_with(".json")) + { + let Some((commit, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), &commit_object) + .await? + else { + return Err(TableCatalogStoreError::Conflict(format!( + "commit log changed while preparing durable strong snapshot: {commit_object}" + ))); + }; + commits.push(StrongCommitSnapshotRecord { + table_bucket: table_bucket.to_string(), + table_id: table_entry.table_id.clone(), + lookup_key: commit.commit_id.clone(), + commit, + }); + } + for idempotency_object in self + .backend + .list_objects( + self.catalog_bucket(), + &self + .paths + .commit_idempotency_entries_prefix(table_bucket, &table_entry.table_id), + ) + .await? + .into_iter() + .filter(|object| object.ends_with(".json")) + { + let Some((commit, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), &idempotency_object) + .await? + else { + return Err(TableCatalogStoreError::Conflict(format!( + "idempotency index changed while preparing durable strong snapshot: {idempotency_object}" + ))); + }; + let lookup_key = commit.idempotency_key.clone().ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("idempotency index {idempotency_object} has no idempotency key")) + })?; + idempotency.push(StrongCommitSnapshotRecord { + table_bucket: table_bucket.to_string(), + table_id: table_entry.table_id.clone(), + lookup_key, + commit, + }); + } + tables.push(table_entry); + } + + let view_objects = self + .backend + .list_objects(self.catalog_bucket(), &self.paths.view_entries_prefix(table_bucket, &namespace)) + .await?; + for view_object in view_objects.iter().filter(|object| object.ends_with(VIEW_ENTRY_FILE)) { + unmatched_view_objects.remove(view_object); + guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), view_object).await?); + let Some((view_entry, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), view_object) + .await? + else { + return Err(TableCatalogStoreError::Conflict(format!( + "view changed while preparing durable strong snapshot: {view_object}" + ))); + }; + if view_entry.table_bucket != table_bucket || view_entry.namespace != namespace_entry.namespace { + return Err(TableCatalogStoreError::Invalid(format!( + "view {} does not match its catalog namespace", + view_entry.view + ))); + } + views.push(view_entry); + } + namespaces.push(namespace_entry); + } + if let Some(object) = unmatched_table_objects.first() { + return Err(TableCatalogStoreError::Invalid(format!( + "table entry has no namespace entry during durable strong migration: {object}" + ))); + } + if let Some(object) = unmatched_view_objects.first() { + return Err(TableCatalogStoreError::Invalid(format!( + "view entry has no namespace entry during durable strong migration: {object}" + ))); + } + + namespaces.sort_by(|left, right| left.namespace.cmp(&right.namespace)); + tables.sort_by(|left, right| (&left.namespace, &left.table).cmp(&(&right.namespace, &right.table))); + views.sort_by(|left, right| (&left.namespace, &left.view).cmp(&(&right.namespace, &right.view))); + commits.sort_by(|left, right| (&left.table_id, &left.lookup_key).cmp(&(&right.table_id, &right.lookup_key))); + idempotency.sort_by(|left, right| (&left.table_id, &left.lookup_key).cmp(&(&right.table_id, &right.lookup_key))); + + let snapshot = StrongTableCatalogBucketSnapshot { + table_bucket: table_bucket_entry, + namespaces, + tables, + views, + commits, + idempotency, + }; + self.validate_bucket_snapshot_for_migration(&snapshot)?; + Ok(snapshot) + } + + fn validate_bucket_snapshot_for_migration(&self, snapshot: &StrongTableCatalogBucketSnapshot) -> TableCatalogStoreResult<()> { + let table_bucket = &snapshot.table_bucket.table_bucket; + let tables_by_id = snapshot + .tables + .iter() + .map(|table| (table.table_id.as_str(), table)) + .collect::>(); + if tables_by_id.len() != snapshot.tables.len() { + return Err(TableCatalogStoreError::Invalid( + "migration snapshot contains duplicate table ids".to_string(), + )); + } + let commits_by_key = snapshot + .commits + .iter() + .map(|record| ((record.table_id.as_str(), record.lookup_key.as_str()), &record.commit)) + .collect::>(); + if commits_by_key.len() != snapshot.commits.len() { + return Err(TableCatalogStoreError::Invalid( + "migration snapshot contains duplicate commit lookup keys".to_string(), + )); + } + let idempotency_by_key = snapshot + .idempotency + .iter() + .map(|record| ((record.table_id.as_str(), record.lookup_key.as_str()), &record.commit)) + .collect::>(); + if idempotency_by_key.len() != snapshot.idempotency.len() { + return Err(TableCatalogStoreError::Invalid( + "migration snapshot contains duplicate idempotency lookup keys".to_string(), + )); + } + for record in &snapshot.commits { + let table = tables_by_id.get(record.table_id.as_str()).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("commit {} has no table in migration snapshot", record.commit.commit_id)) + })?; + if record.table_bucket != *table_bucket + || record.commit.table_id != record.table_id + || record.lookup_key != record.commit.commit_id + { + return Err(TableCatalogStoreError::Invalid(format!( + "commit {} does not match its migration snapshot owner", + record.commit.commit_id + ))); + } + let indexed = record + .commit + .idempotency_key + .as_deref() + .and_then(|idempotency_key| idempotency_by_key.get(&(record.table_id.as_str(), idempotency_key)).copied()); + let recovery = table_commit_recovery_entry(table, &record.commit, indexed); + if recovery.recovery_state != TableCommitRecoveryState::Committed { + return Err(TableCatalogStoreError::Conflict(format!( + "commit {} requires catalog recovery before durable strong migration", + record.commit.commit_id + ))); + } + } + for record in &snapshot.idempotency { + let _table = tables_by_id.get(record.table_id.as_str()).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "idempotency index {} has no table in migration snapshot", + record.lookup_key + )) + })?; + if record.table_bucket != *table_bucket || record.commit.table_id != record.table_id { + return Err(TableCatalogStoreError::Invalid(format!( + "idempotency index {} does not match its migration snapshot owner", + record.lookup_key + ))); + } + if record.commit.idempotency_key.as_deref() != Some(record.lookup_key.as_str()) { + return Err(TableCatalogStoreError::Invalid(format!( + "idempotency index {} does not match its commit payload", + record.lookup_key + ))); + } + let committed = commits_by_key + .get(&(record.table_id.as_str(), record.commit.commit_id.as_str())) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "idempotency index {} has no commit record in migration snapshot", + record.lookup_key + )) + })?; + if *committed != &record.commit { + return Err(TableCatalogStoreError::Conflict(format!( + "idempotency index {} requires catalog recovery before durable strong migration", + record.lookup_key + ))); + } + } + + let mut state = StrongTableCatalogState { + hydrated: true, + ..StrongTableCatalogState::default() + }; + StrongTableCatalogStore::::insert_bucket_snapshot_locked(&mut state, snapshot.clone())?; + if state.namespaces.len() != snapshot.namespaces.len() + || state.tables.len() != snapshot.tables.len() + || state.views.len() != snapshot.views.len() + || state.commits.len() != snapshot.commits.len() + || state.idempotency.len() != snapshot.idempotency.len() + { + return Err(TableCatalogStoreError::Invalid( + "migration snapshot contains duplicate catalog identities".to_string(), + )); + } + Ok(()) + } + + pub(crate) async fn plan_durable_strong_backing_migration( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult { + if self.get_table_bucket(table_bucket).await?.is_none() { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + } + + let namespaces = self.list_namespaces(table_bucket).await?; + let mut table_count: usize = 0; + let mut view_count: usize = 0; + let mut commit_log_count: usize = 0; + let mut idempotency_index_count: usize = 0; + let mut recovery_required_count: usize = 0; + let mut manual_review_count: usize = 0; + let mut warehouse_prefix_owners = BTreeMap::::new(); + + for namespace in &namespaces { + let tables = self.list_tables(table_bucket, &namespace.namespace).await?; + for table in tables { + table_count += 1; + if table.state == TableCatalogEntryState::Active { + let warehouse_prefix = table_warehouse_object_prefix(&table)?; + warehouse_prefix_owners + .entry(warehouse_prefix) + .and_modify(|count| *count = count.saturating_add(1)) + .or_insert(1); + } + + let recovery = self.table_commit_recovery_report_for_entry(&table, 0).await?; + commit_log_count = commit_log_count.saturating_add(recovery.commits.len()); + idempotency_index_count = idempotency_index_count.saturating_add( + self.backend + .list_objects( + self.catalog_bucket(), + &self.paths.commit_idempotency_entries_prefix(table_bucket, &table.table_id), + ) + .await? + .into_iter() + .filter(|object| object.ends_with(".json")) + .count(), + ); + recovery_required_count = recovery_required_count + .saturating_add(recovery.staged_before_table_update_count) + .saturating_add(recovery.finalization_required_count) + .saturating_add(recovery.idempotency_repair_required_count); + manual_review_count = manual_review_count.saturating_add(recovery.manual_review_count); + } + view_count = view_count.saturating_add(self.list_views(table_bucket, &namespace.namespace).await?.len()); + } + + let warehouse_index_ready = self.warehouse_index_ready(table_bucket).await?; + let duplicate_warehouse_prefix_count = warehouse_prefix_owners.values().filter(|count| **count > 1).count(); + let mut blockers = Vec::new(); + let mut recommended_actions = Vec::new(); + if recovery_required_count > 0 { + blockers.push(TableCatalogBackingMigrationBlocker::CommitRecoveryRequired); + } + if manual_review_count > 0 { + blockers.push(TableCatalogBackingMigrationBlocker::CommitManualReviewRequired); + } + if recovery_required_count > 0 || manual_review_count > 0 { + recommended_actions.push(TableCatalogBackingMigrationAction::RunCatalogRecovery); + } + if !warehouse_index_ready { + blockers.push(TableCatalogBackingMigrationBlocker::WarehouseIndexBackfillRequired); + recommended_actions.push(TableCatalogBackingMigrationAction::BackfillWarehouseIndex); + } + if duplicate_warehouse_prefix_count > 0 { + blockers.push(TableCatalogBackingMigrationBlocker::DuplicateWarehousePrefix); + recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateWarehousePrefixes); + } + + let mut status = if manual_review_count > 0 || duplicate_warehouse_prefix_count > 0 { + TableCatalogBackingMigrationStatus::ManualReviewRequired + } else if recovery_required_count > 0 || !warehouse_index_ready { + TableCatalogBackingMigrationStatus::RecoveryRequired + } else { + TableCatalogBackingMigrationStatus::ReadyToSnapshot + }; + + let strong_store = StrongTableCatalogStore::new(self.backend.clone()); + let migration_fence = self.read_backing_migration_fence(table_bucket).await?.map(|(fence, _)| fence); + let object_backed_writes_fenced = migration_fence.is_some(); + if status == TableCatalogBackingMigrationStatus::ReadyToSnapshot + && let Some(fence) = migration_fence.as_ref() + && fence.status == TableCatalogBackingMigrationFenceStatus::Materialized + && let Some(source_fingerprint) = fence.source_fingerprint.as_deref() + { + if strong_store.bucket_snapshot_fingerprint(table_bucket).await?.as_deref() == Some(source_fingerprint) { + status = TableCatalogBackingMigrationStatus::SnapshotMaterialized; + } else { + status = TableCatalogBackingMigrationStatus::ManualReviewRequired; + blockers.push(TableCatalogBackingMigrationBlocker::DurableStrongSnapshotChanged); + recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDurableStrongSnapshot); + } + } + + let ready_to_enable_durable_strong = status == TableCatalogBackingMigrationStatus::SnapshotMaterialized + && self.all_table_buckets_materialized(&strong_store).await?; + if status == TableCatalogBackingMigrationStatus::ReadyToSnapshot { + recommended_actions.extend([ + TableCatalogBackingMigrationAction::SnapshotObjectBackedCatalog, + TableCatalogBackingMigrationAction::KeepObjectBackedRollbackConfig, + ]); + } else if status == TableCatalogBackingMigrationStatus::SnapshotMaterialized { + recommended_actions.push(TableCatalogBackingMigrationAction::VerifyDurableStrongSnapshot); + if ready_to_enable_durable_strong { + recommended_actions.push(TableCatalogBackingMigrationAction::EnableDurableStrongBacking); + } else { + recommended_actions.push(TableCatalogBackingMigrationAction::SnapshotRemainingTableBuckets); + } + recommended_actions.push(TableCatalogBackingMigrationAction::KeepObjectBackedRollbackConfig); + } + + Ok(TableCatalogBackingMigrationDryRunReport { + table_bucket: table_bucket.to_string(), + source_kind: TableCatalogBackingKind::ObjectBacked, + target_kind: TableCatalogBackingKind::StrongKvWal, + status, + namespace_count: namespaces.len(), + table_count, + view_count, + commit_log_count, + idempotency_index_count, + warehouse_prefix_count: warehouse_prefix_owners.len(), + warehouse_index_ready, + object_backed_writes_fenced, + ready_to_enable_durable_strong, + blockers, + recommended_actions, + rollback: TableCatalogBackingRollbackPlan { + backing_config_key: ENV_TABLE_CATALOG_BACKING, + current_backing_value: TABLE_CATALOG_BACKING_DURABLE_STRONG, + rollback_backing_value: TABLE_CATALOG_BACKING_OBJECT, + preserves_object_backed_catalog: true, + requires_operator_restart: true, + }, + }) + } + + pub(crate) async fn materialize_durable_strong_backing_migration( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult { + let fence_path = self.paths.backing_migration_fence_path(table_bucket); + let fence_lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); + let global_fence_path = self.paths.backing_migration_global_fence_path(); + let global_fence_lock_path = self.paths.backing_migration_global_fence_lock_path(); + if self.read_backing_migration_fence(table_bucket).await?.is_none() { + let preflight = self.plan_durable_strong_backing_migration(table_bucket).await?; + if preflight.status != TableCatalogBackingMigrationStatus::ReadyToSnapshot { + return Err(TableCatalogStoreError::Conflict(format!( + "table bucket {table_bucket} is not ready for durable strong snapshot materialization" + ))); + } + } + + let _global_fence_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &global_fence_lock_path) + .await?; + let _fence_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &fence_lock_path) + .await?; + let existing_fence = self.read_backing_migration_fence(table_bucket).await?; + let strong_store = StrongTableCatalogStore::new(self.backend.clone()); + if let Some((fence, _)) = existing_fence.as_ref() + && (fence.version != TABLE_CATALOG_MIGRATION_VERSION || fence.table_bucket != table_bucket) + { + return Err(TableCatalogStoreError::Invalid(format!( + "invalid durable strong migration fence for table bucket {table_bucket}" + ))); + } + + if !self.warehouse_index_ready(table_bucket).await? { + return Err(TableCatalogStoreError::Conflict(format!( + "table bucket {table_bucket} warehouse index must be backfilled before durable strong migration" + ))); + } + + let mut source_guards = Vec::new(); + let source = self + .collect_bucket_snapshot_with_locks(table_bucket, &mut source_guards) + .await?; + let source_fingerprint = table_catalog_bucket_snapshot_fingerprint(&source)?; + if let Some((fence, _)) = existing_fence.as_ref() + && fence.status == TableCatalogBackingMigrationFenceStatus::Materialized + && fence.source_fingerprint.as_deref() != Some(source_fingerprint.as_str()) + { + return Err(TableCatalogStoreError::Conflict(format!( + "object-backed catalog state no longer matches the materialized snapshot for table bucket {table_bucket}" + ))); + } + + self.ensure_global_backing_migration_fence(&global_fence_path).await?; + let (migration_id, target_bucket_existed) = if let Some((fence, _)) = existing_fence.as_ref() { + (fence.migration_id.clone(), fence.target_bucket_existed) + } else { + let target_bucket_existed = strong_store.bucket_snapshot_fingerprint(table_bucket).await?.is_some(); + let fence = TableCatalogBackingMigrationFence { + version: TABLE_CATALOG_MIGRATION_VERSION, + table_bucket: table_bucket.to_string(), + migration_id: Uuid::new_v4().to_string(), + status: TableCatalogBackingMigrationFenceStatus::Preparing, + target_bucket_existed, + source_fingerprint: None, + target_snapshot_etag: None, + }; + self.write_entry(self.catalog_bucket(), &fence_path, &fence, TableCatalogPutPrecondition::IfAbsent) + .await?; + (fence.migration_id, target_bucket_existed) + }; + + let (target_snapshot_etag, created) = strong_store.materialize_bucket_snapshot(source.clone()).await?; + let completed_fence = TableCatalogBackingMigrationFence { + version: TABLE_CATALOG_MIGRATION_VERSION, + table_bucket: table_bucket.to_string(), + migration_id, + status: TableCatalogBackingMigrationFenceStatus::Materialized, + target_bucket_existed, + source_fingerprint: Some(source_fingerprint.clone()), + target_snapshot_etag: Some(target_snapshot_etag.clone()), + }; + self.write_entry(self.catalog_bucket(), &fence_path, &completed_fence, TableCatalogPutPrecondition::Any) + .await?; + + drop(source_guards); + drop(_fence_guard); + let ready_to_enable_durable_strong = self.all_table_buckets_materialized(&strong_store).await?; + Ok(TableCatalogBackingMigrationExecutionReport { + table_bucket: table_bucket.to_string(), + source_kind: TableCatalogBackingKind::ObjectBacked, + target_kind: TableCatalogBackingKind::StrongKvWal, + status: if created { + TableCatalogBackingMigrationExecutionStatus::SnapshotMaterialized + } else { + TableCatalogBackingMigrationExecutionStatus::SnapshotAlreadyMaterialized + }, + namespace_count: source.namespaces.len(), + table_count: source.tables.len(), + view_count: source.views.len(), + commit_log_count: source.commits.len(), + idempotency_index_count: source.idempotency.len(), + source_fingerprint, + target_snapshot_etag, + object_backed_writes_fenced: true, + ready_to_enable_durable_strong, + }) + } + + pub(crate) async fn cancel_durable_strong_backing_migration( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult { + let fence_path = self.paths.backing_migration_fence_path(table_bucket); + let fence_lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); + let global_fence_path = self.paths.backing_migration_global_fence_path(); + let global_fence_lock_path = self.paths.backing_migration_global_fence_lock_path(); + let _global_fence_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &global_fence_lock_path) + .await?; + let _fence_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &fence_lock_path) + .await?; + let Some((fence, _)) = self.read_backing_migration_fence(table_bucket).await? else { + self.clear_global_backing_migration_fence_if_unused(&global_fence_path) + .await?; + return Ok(TableCatalogBackingMigrationCancelReport { + table_bucket: table_bucket.to_string(), + status: TableCatalogBackingMigrationCancelStatus::NoMigrationFence, + object_backed_writes_fenced: false, + }); + }; + self.ensure_global_backing_migration_fence(&global_fence_path).await?; + if fence.version != TABLE_CATALOG_MIGRATION_VERSION || fence.table_bucket != table_bucket { + return Err(TableCatalogStoreError::Invalid(format!( + "invalid durable strong migration fence for table bucket {table_bucket}" + ))); + } + + let mut source_guards = Vec::new(); + let source = self + .collect_bucket_snapshot_with_locks(table_bucket, &mut source_guards) + .await?; + let source_fingerprint = table_catalog_bucket_snapshot_fingerprint(&source)?; + if fence.status == TableCatalogBackingMigrationFenceStatus::Materialized + && fence.source_fingerprint.as_deref() != Some(source_fingerprint.as_str()) + { + return Err(TableCatalogStoreError::Conflict(format!( + "object-backed catalog state changed after materializing table bucket {table_bucket}" + ))); + } + + let strong_store = StrongTableCatalogStore::new(self.backend.clone()); + if fence.status == TableCatalogBackingMigrationFenceStatus::Materialized + && strong_store.bucket_snapshot_fingerprint(table_bucket).await?.as_deref() != Some(&source_fingerprint) + { + return Err(TableCatalogStoreError::Conflict(format!( + "durable strong catalog state changed after materializing table bucket {table_bucket}" + ))); + } + if !fence.target_bucket_existed { + strong_store + .remove_bucket_snapshot_if_unchanged(table_bucket, &source_fingerprint) + .await?; + } + self.backend.delete_object(self.catalog_bucket(), &fence_path).await?; + self.clear_global_backing_migration_fence_if_unused(&global_fence_path) + .await?; + Ok(TableCatalogBackingMigrationCancelReport { + table_bucket: table_bucket.to_string(), + status: TableCatalogBackingMigrationCancelStatus::FenceReleased, + object_backed_writes_fenced: false, + }) + } + + async fn all_table_buckets_materialized(&self, strong_store: &StrongTableCatalogStore) -> TableCatalogStoreResult { + if self + .read_entry::( + self.catalog_bucket(), + &self.paths.backing_migration_global_fence_path(), + ) + .await? + .is_none() + { + return Ok(false); + } + let table_bucket_objects = self + .backend + .list_objects(self.catalog_bucket(), &self.paths.table_bucket_entries_prefix()) + .await?; + for table_bucket_object in table_bucket_objects + .iter() + .filter(|object| object.ends_with(TABLE_BUCKET_ENTRY_FILE)) + { + let Some((entry, _)) = self + .read_entry::(self.catalog_bucket(), table_bucket_object) + .await? + else { + return Ok(false); + }; + let Some((fence, _)) = self.read_backing_migration_fence(&entry.table_bucket).await? else { + return Ok(false); + }; + if fence.status != TableCatalogBackingMigrationFenceStatus::Materialized { + return Ok(false); + } + let Some(source_fingerprint) = fence.source_fingerprint.as_deref() else { + return Ok(false); + }; + if strong_store + .bucket_snapshot_fingerprint(&entry.table_bucket) + .await? + .as_deref() + != Some(source_fingerprint) + { + return Ok(false); + } + } + Ok(true) + } +} diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs new file mode 100644 index 000000000..3b5c3559b --- /dev/null +++ b/rustfs/src/table_catalog/store/mod.rs @@ -0,0 +1,1173 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +mod migration; +mod object; +mod strong; + +use migration::table_catalog_backing_manifest; +pub(crate) use object::ObjectTableCatalogStore; +#[cfg(test)] +pub(super) use strong::StrongTableCatalogSnapshot; +pub(crate) use strong::StrongTableCatalogStore; + +#[async_trait::async_trait] +pub(crate) trait TableCatalogStore: Send + Sync { + async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult>; + + async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()>; + + async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()>; + + async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult>; + + async fn list_namespaces_page( + &self, + table_bucket: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + Ok(catalog_list_page_from_entries( + self.list_namespaces(table_bucket).await?, + cursor, + limit, + |entry| &entry.namespace, + )) + } + + async fn get_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; + + async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()>; + + async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()>; + + async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()>; + + async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; + + async fn list_tables_page( + &self, + table_bucket: &str, + namespace: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + Ok(catalog_list_page_from_entries( + self.list_tables(table_bucket, namespace).await?, + cursor, + limit, + |entry| &entry.table, + )) + } + + async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult>; + + async fn resolve_table_data_plane_resource( + &self, + table_bucket: &str, + object: &str, + ) -> TableCatalogStoreResult> { + scan_table_data_plane_resource_for_object(self, table_bucket, object).await + } + + async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult; + + async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()>; + + async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()>; + + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; + + async fn list_views_page( + &self, + table_bucket: &str, + namespace: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + Ok(catalog_list_page_from_entries( + self.list_views(table_bucket, namespace).await?, + cursor, + limit, + |entry| &entry.view, + )) + } + + async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult>; + + async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult; + + async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()>; + + async fn get_commit_by_id( + &self, + table_bucket: &str, + table_id: &str, + commit_id: &str, + ) -> TableCatalogStoreResult>; + + async fn get_commit_by_idempotency_key( + &self, + table_bucket: &str, + table_id: &str, + idempotency_key: &str, + ) -> TableCatalogStoreResult>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TableCatalogObject { + pub data: Vec, + pub etag: Option, + pub mod_time: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TableCatalogObjectMetadata { + pub etag: Option, + pub mod_time: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TableCatalogObjectListPage { + pub objects: Vec, + pub is_truncated: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum TableCatalogPutPrecondition { + Any, + IfAbsent, + IfMatch(String), +} + +#[async_trait::async_trait] +pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { + async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; + + async fn read_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + self.read_object(bucket, object).await + } + + async fn object_metadata(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + Ok(self + .read_object(bucket, object) + .await? + .map(|object| TableCatalogObjectMetadata { + etag: object.etag, + mod_time: object.mod_time, + })) + } + + async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult; + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()>; + + async fn put_object_unlocked( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> { + self.put_object(bucket, object, data, precondition).await + } + + async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()>; + + async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { + self.delete_object(bucket, object).await + } + + async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult>; + + async fn list_objects_page( + &self, + bucket: &str, + prefix: &str, + start_after: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult { + let mut objects = self.list_objects(bucket, prefix).await?; + objects.sort(); + let start = start_after.map_or(0, |cursor| objects.partition_point(|object| object.as_str() <= cursor)); + let mut objects = objects + .into_iter() + .skip(start) + .take(limit.get().saturating_add(1)) + .collect::>(); + let is_truncated = objects.len() > limit.get(); + objects.truncate(limit.get()); + Ok(TableCatalogObjectListPage { objects, is_truncated }) + } + + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + self.acquire_write_lock(bucket, object).await + } + + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TableCatalogObjectPaths { + catalog_root: &'static str, +} + +impl Default for TableCatalogObjectPaths { + fn default() -> Self { + Self { + catalog_root: INTERNAL_CATALOG_ROOT, + } + } +} + +impl TableCatalogObjectPaths { + pub fn table_bucket_entries_prefix(&self) -> String { + format!("{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT) + } + + pub fn table_bucket_entry_path(&self, table_bucket: &str) -> String { + format!("{}{}", self.table_bucket_root_prefix(table_bucket), TABLE_BUCKET_ENTRY_FILE) + } + + pub fn table_bucket_maintenance_config_path(&self, table_bucket: &str) -> String { + format!( + "{}{MAINTENANCE_ROOT}/{MAINTENANCE_CONFIG_FILE}", + self.table_bucket_root_prefix(table_bucket) + ) + } + + pub fn namespace_entries_prefix(&self, table_bucket: &str) -> String { + format!("{}{}/", self.table_bucket_root_prefix(table_bucket), NAMESPACE_ROOT) + } + + pub fn namespace_entry_path(&self, table_bucket: &str, namespace: &Namespace) -> String { + format!( + "{}{}/{}", + self.namespace_entries_prefix(table_bucket), + namespace.storage_id(), + NAMESPACE_ENTRY_FILE + ) + } + + pub fn table_entries_prefix(&self, table_bucket: &str, namespace: &Namespace) -> String { + format!( + "{}{}/{}/", + self.namespace_entries_prefix(table_bucket), + namespace.storage_id(), + TABLE_ROOT + ) + } + + pub fn table_entry_path(&self, table_bucket: &str, namespace: &Namespace, table: &IdentifierSegment) -> String { + format!( + "{}{}/{}", + self.table_entries_prefix(table_bucket, namespace), + table.as_str(), + TABLE_ENTRY_FILE + ) + } + + pub fn external_catalog_bridge_path(&self, table_bucket: &str, namespace: &Namespace, table: &IdentifierSegment) -> String { + format!( + "{}{}/{EXTERNAL_CATALOG_ROOT}/{EXTERNAL_CATALOG_BRIDGE_FILE}", + self.table_entries_prefix(table_bucket, namespace), + table.as_str() + ) + } + + pub fn view_entries_prefix(&self, table_bucket: &str, namespace: &Namespace) -> String { + format!("{}{}/{}/", self.namespace_entries_prefix(table_bucket), namespace.storage_id(), VIEW_ROOT) + } + + pub fn view_entry_path(&self, table_bucket: &str, namespace: &Namespace, view: &IdentifierSegment) -> String { + format!( + "{}{}/{}", + self.view_entries_prefix(table_bucket, namespace), + view.as_str(), + VIEW_ENTRY_FILE + ) + } + + pub fn table_maintenance_config_path( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, + ) -> String { + format!( + "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CONFIG_FILE}", + self.table_entries_prefix(table_bucket, namespace), + table.as_str(), + table_catalog_path_hash(table_id) + ) + } + + pub fn table_maintenance_job_path( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, + job_id: &str, + ) -> String { + format!( + "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_JOB_ROOT}/{}.json", + self.table_entries_prefix(table_bucket, namespace), + table.as_str(), + table_catalog_path_hash(table_id), + table_catalog_path_hash(job_id) + ) + } + + pub fn table_maintenance_jobs_prefix( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, + ) -> String { + format!( + "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_JOB_ROOT}/", + self.table_entries_prefix(table_bucket, namespace), + table.as_str(), + table_catalog_path_hash(table_id) + ) + } + + pub fn table_maintenance_latest_job_path( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, + ) -> String { + format!( + "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_LATEST_JOB_FILE}", + self.table_entries_prefix(table_bucket, namespace), + table.as_str(), + table_catalog_path_hash(table_id) + ) + } + + pub fn table_maintenance_current_job_path( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, + ) -> String { + format!( + "{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CURRENT_JOB_FILE}", + self.table_entries_prefix(table_bucket, namespace), + table.as_str(), + table_catalog_path_hash(table_id) + ) + } + + pub fn commit_log_entry_path(&self, table_bucket: &str, table_id: &str, commit_id: &str) -> String { + format!( + "{}{}/{}/{}.json", + self.table_bucket_root_prefix(table_bucket), + COMMIT_LOG_ROOT, + table_catalog_path_hash(table_id), + table_catalog_path_hash(commit_id) + ) + } + + pub fn commit_log_entries_prefix(&self, table_bucket: &str, table_id: &str) -> String { + format!( + "{}{}/{}/", + self.table_bucket_root_prefix(table_bucket), + COMMIT_LOG_ROOT, + table_catalog_path_hash(table_id) + ) + } + + pub fn commit_idempotency_entry_path(&self, table_bucket: &str, table_id: &str, idempotency_key: &str) -> String { + format!( + "{}{}/{}/{}.json", + self.table_bucket_root_prefix(table_bucket), + COMMIT_IDEMPOTENCY_ROOT, + table_catalog_path_hash(table_id), + table_catalog_path_hash(idempotency_key) + ) + } + + pub fn commit_idempotency_entries_prefix(&self, table_bucket: &str, table_id: &str) -> String { + format!( + "{}{}/{}/", + self.table_bucket_root_prefix(table_bucket), + COMMIT_IDEMPOTENCY_ROOT, + table_catalog_path_hash(table_id) + ) + } + + pub fn warehouse_index_state_path(&self, table_bucket: &str) -> String { + format!( + "{}{}/{}", + self.table_bucket_root_prefix(table_bucket), + WAREHOUSE_INDEX_ROOT, + WAREHOUSE_INDEX_STATE_FILE + ) + } + + pub fn warehouse_index_entry_path(&self, table_bucket: &str, warehouse_object_prefix: &str) -> String { + format!( + "{}{}/{}.json", + self.table_bucket_root_prefix(table_bucket), + WAREHOUSE_INDEX_ROOT, + table_catalog_path_hash(warehouse_object_prefix) + ) + } + + pub fn backing_migration_fence_path(&self, table_bucket: &str) -> String { + format!( + "{}{}/{}", + self.table_bucket_root_prefix(table_bucket), + TABLE_CATALOG_MIGRATION_ROOT, + TABLE_CATALOG_MIGRATION_FENCE_FILE + ) + } + + pub fn backing_migration_fence_lock_path(&self, table_bucket: &str) -> String { + format!( + "{}{}/{}", + self.table_bucket_root_prefix(table_bucket), + TABLE_CATALOG_MIGRATION_ROOT, + TABLE_CATALOG_MIGRATION_FENCE_LOCK + ) + } + + pub fn backing_migration_global_fence_path(&self) -> String { + format!( + "{}/{}/{}", + self.catalog_root, TABLE_CATALOG_MIGRATION_ROOT, TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_FILE + ) + } + + pub fn backing_migration_global_fence_lock_path(&self) -> String { + format!( + "{}/{}/{}", + self.catalog_root, TABLE_CATALOG_MIGRATION_ROOT, TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_LOCK + ) + } + + fn table_bucket_root_prefix(&self, table_bucket: &str) -> String { + format!("{}/{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT, table_catalog_path_hash(table_bucket)) + } +} + +#[derive(Clone)] +pub(crate) enum ConfiguredTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + ObjectBacked(ObjectTableCatalogStore), + DurableStrong(StrongTableCatalogStore), +} + +impl ConfiguredTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + pub(crate) fn from_env(backend: B) -> TableCatalogStoreResult { + Ok(Self::new(backend, TableCatalogBackingMode::from_env()?)) + } + + pub(crate) fn new(backend: B, mode: TableCatalogBackingMode) -> Self { + match mode { + TableCatalogBackingMode::ObjectBacked => Self::ObjectBacked(ObjectTableCatalogStore::new(backend)), + TableCatalogBackingMode::DurableStrong => Self::DurableStrong(StrongTableCatalogStore::new(backend)), + } + } + + pub(crate) fn backing_mode(&self) -> TableCatalogBackingMode { + match self { + Self::ObjectBacked(_) => TableCatalogBackingMode::ObjectBacked, + Self::DurableStrong(_) => TableCatalogBackingMode::DurableStrong, + } + } + + fn unsupported_for_durable_strong(operation: &str) -> TableCatalogStoreError { + TableCatalogStoreError::Invalid(format!( + "{operation} is not supported with {TABLE_CATALOG_BACKING_DURABLE_STRONG} table catalog backing" + )) + } +} + +#[async_trait::async_trait] +impl TableCatalogStore for ConfiguredTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.get_table_bucket(table_bucket).await, + Self::DurableStrong(store) => store.get_table_bucket(table_bucket).await, + } + } + + async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.put_table_bucket(entry).await, + Self::DurableStrong(store) => store.put_table_bucket(entry).await, + } + } + + async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.create_namespace(entry).await, + Self::DurableStrong(store) => store.create_namespace(entry).await, + } + } + + async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.list_namespaces(table_bucket).await, + Self::DurableStrong(store) => store.list_namespaces(table_bucket).await, + } + } + + async fn list_namespaces_page( + &self, + table_bucket: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.list_namespaces_page(table_bucket, cursor, limit).await, + Self::DurableStrong(store) => store.list_namespaces_page(table_bucket, cursor, limit).await, + } + } + + async fn get_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.get_namespace(table_bucket, namespace).await, + Self::DurableStrong(store) => store.get_namespace(table_bucket, namespace).await, + } + } + + async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.drop_namespace(table_bucket, namespace).await, + Self::DurableStrong(store) => store.drop_namespace(table_bucket, namespace).await, + } + } + + async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.create_table(entry).await, + Self::DurableStrong(store) => store.create_table(entry).await, + } + } + + async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.register_table(entry).await, + Self::DurableStrong(store) => store.register_table(entry).await, + } + } + + async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.list_tables(table_bucket, namespace).await, + Self::DurableStrong(store) => store.list_tables(table_bucket, namespace).await, + } + } + + async fn list_tables_page( + &self, + table_bucket: &str, + namespace: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.list_tables_page(table_bucket, namespace, cursor, limit).await, + Self::DurableStrong(store) => store.list_tables_page(table_bucket, namespace, cursor, limit).await, + } + } + + async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.load_table(table_bucket, namespace, table).await, + Self::DurableStrong(store) => store.load_table(table_bucket, namespace, table).await, + } + } + + async fn resolve_table_data_plane_resource( + &self, + table_bucket: &str, + object: &str, + ) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.resolve_table_data_plane_resource(table_bucket, object).await, + Self::DurableStrong(store) => store.resolve_table_data_plane_resource(table_bucket, object).await, + } + } + + async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => store.commit_table(request).await, + Self::DurableStrong(store) => store.commit_table(request).await, + } + } + + async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.drop_table(table_bucket, namespace, table).await, + Self::DurableStrong(store) => store.drop_table(table_bucket, namespace, table).await, + } + } + + async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.create_view(entry).await, + Self::DurableStrong(store) => store.create_view(entry).await, + } + } + + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.list_views(table_bucket, namespace).await, + Self::DurableStrong(store) => store.list_views(table_bucket, namespace).await, + } + } + + async fn list_views_page( + &self, + table_bucket: &str, + namespace: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.list_views_page(table_bucket, namespace, cursor, limit).await, + Self::DurableStrong(store) => store.list_views_page(table_bucket, namespace, cursor, limit).await, + } + } + + async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.load_view(table_bucket, namespace, view).await, + Self::DurableStrong(store) => store.load_view(table_bucket, namespace, view).await, + } + } + + async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => store.replace_view(request).await, + Self::DurableStrong(store) => store.replace_view(request).await, + } + } + + async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.drop_view(table_bucket, namespace, view).await, + Self::DurableStrong(store) => store.drop_view(table_bucket, namespace, view).await, + } + } + + async fn get_commit_by_id( + &self, + table_bucket: &str, + table_id: &str, + commit_id: &str, + ) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.get_commit_by_id(table_bucket, table_id, commit_id).await, + Self::DurableStrong(store) => store.get_commit_by_id(table_bucket, table_id, commit_id).await, + } + } + + async fn get_commit_by_idempotency_key( + &self, + table_bucket: &str, + table_id: &str, + idempotency_key: &str, + ) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => { + store + .get_commit_by_idempotency_key(table_bucket, table_id, idempotency_key) + .await + } + Self::DurableStrong(store) => { + store + .get_commit_by_idempotency_key(table_bucket, table_id, idempotency_key) + .await + } + } + } +} + +impl ConfiguredTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + pub(crate) async fn get_table_maintenance_config( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => store.get_table_maintenance_config(table_bucket, namespace, table).await, + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance config")), + } + } + + pub(crate) async fn put_table_maintenance_config( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + config: TableMaintenanceConfig, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .put_table_maintenance_config(table_bucket, namespace, table, config) + .await + } + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance config")), + } + } + + pub(crate) async fn get_table_metadata_maintenance_report( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + job_id: &str, + ) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => { + store + .get_table_metadata_maintenance_report(table_bucket, namespace, table, job_id) + .await + } + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance report")), + } + } + + pub(crate) async fn get_table_maintenance_scheduler_report( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .get_table_maintenance_scheduler_report(table_bucket, namespace, table) + .await + } + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance scheduler")), + } + } + + pub(crate) async fn run_table_maintenance_scheduler_once( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + scheduler_id: String, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .run_table_maintenance_scheduler_once(table_bucket, namespace, table, scheduler_id) + .await + } + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance scheduler")), + } + } + + pub(crate) async fn apply_table_maintenance_quarantine_operation( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + job_id: &str, + request: TableMaintenanceQuarantineOperationRequest, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .apply_table_maintenance_quarantine_operation(table_bucket, namespace, table, job_id, request) + .await + } + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance quarantine")), + } + } + + pub(crate) async fn run_table_metadata_maintenance_worker_once( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + worker_id: String, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .run_table_metadata_maintenance_worker_once(table_bucket, namespace, table, worker_id) + .await + } + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance worker")), + } + } + + pub(crate) async fn heartbeat_table_metadata_maintenance_job( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + job_id: &str, + lease_id: &str, + worker_id: &str, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .heartbeat_table_metadata_maintenance_job(table_bucket, namespace, table, job_id, lease_id, worker_id) + .await + } + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("table maintenance heartbeat")), + } + } + + pub(crate) async fn export_table_catalog_entry( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => store.export_table_catalog_entry(table_bucket, namespace, table).await, + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("catalog export")), + } + } + + pub(crate) async fn diagnose_table_catalog( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + retain_recent_metadata_files: usize, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .diagnose_table_catalog(table_bucket, namespace, table, retain_recent_metadata_files) + .await + } + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("catalog diagnostics")), + } + } + + pub(crate) async fn recover_table_commits( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => store.recover_table_commits(table_bucket, namespace, table).await, + Self::DurableStrong(store) => store.plan_table_commit_recovery(table_bucket, namespace, table).await, + } + } + + pub(crate) async fn get_external_catalog_bridge( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult> { + match self { + Self::ObjectBacked(store) => store.get_external_catalog_bridge(table_bucket, namespace, table).await, + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("external catalog bridge")), + } + } + + pub(crate) async fn put_external_catalog_bridge( + &self, + entry: ExternalCatalogBridgeEntry, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => store.put_external_catalog_bridge(entry).await, + Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("external catalog bridge")), + } + } +} + +pub(crate) struct EcStoreTableCatalogObjectBackend { + store: Arc, +} + +impl Clone for EcStoreTableCatalogObjectBackend { + fn clone(&self) -> Self { + Self { + store: self.store.clone(), + } + } +} + +impl EcStoreTableCatalogObjectBackend +where + S: TableCatalogStorage, +{ + pub fn new(store: Arc) -> Self { + Self { store } + } +} + +pub(crate) type EcStoreTableCatalogStore = ConfiguredTableCatalogStore>; + +#[async_trait::async_trait] +impl TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend +where + S: TableCatalogStorage, +{ + async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + self.read_object_with_options(bucket, object, ObjectOptions::default()).await + } + + async fn read_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + self.read_object_with_options( + bucket, + object, + ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + } + + async fn object_metadata(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + match self.store.get_object_info(bucket, object, &ObjectOptions::default()).await { + Ok(info) => Ok(Some(TableCatalogObjectMetadata { + etag: info.etag, + mod_time: info.mod_time, + })), + Err(err) if is_missing_storage_error(&err) => Ok(None), + Err(err) => Err(storage_error_to_catalog("stat catalog object", err)), + } + } + + async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { + match self.store.get_object_info(bucket, object, &ObjectOptions::default()).await { + Ok(_) => Ok(true), + Err(err) if is_missing_storage_error(&err) => Ok(false), + Err(err) => Err(storage_error_to_catalog("check catalog object", err)), + } + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> { + self.put_object_with_options(bucket, object, data, precondition, false).await + } + + async fn put_object_unlocked( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> { + self.put_object_with_options(bucket, object, data, precondition, true).await + } + + async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { + match self.store.delete_object(bucket, object, ObjectOptions::default()).await { + Ok(_) => Ok(()), + Err(err) if is_missing_storage_error(&err) => Ok(()), + Err(err) => Err(storage_error_to_catalog("delete catalog object", err)), + } + } + + async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { + match self + .store + .delete_object( + bucket, + object, + ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => Ok(()), + Err(err) if is_missing_storage_error(&err) => Ok(()), + Err(err) => Err(storage_error_to_catalog("delete catalog object", err)), + } + } + + async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult> { + let mut continuation = None; + let mut objects = BTreeSet::new(); + let max_keys = i32::try_from(TABLE_CATALOG_LIST_MAX_KEYS) + .map_err(|_| TableCatalogStoreError::Internal("catalog list limit exceeds storage API range".to_string()))?; + + loop { + let result = self + .store + .clone() + .list_objects_v2(bucket, prefix, continuation, None, max_keys, false, None, false) + .await + .map_err(|err| storage_error_to_catalog("list catalog objects", err))?; + + for object in result.objects { + objects.insert(object.name); + } + + if !result.is_truncated { + break; + } + + let Some(next) = result.next_continuation_token else { + break; + }; + continuation = Some(next); + } + + Ok(objects.into_iter().collect()) + } + + async fn list_objects_page( + &self, + bucket: &str, + prefix: &str, + start_after: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult { + let max_keys = i32::try_from(limit.get()) + .map_err(|_| TableCatalogStoreError::Invalid("catalog page size exceeds storage API range".to_string()))?; + let result = self + .store + .clone() + .list_objects_v2(bucket, prefix, None, None, max_keys, false, start_after.map(str::to_string), false) + .await + .map_err(|err| storage_error_to_catalog("list catalog object page", err))?; + let is_truncated = result.is_truncated; + let objects = result.objects.into_iter().map(|object| object.name).collect::>(); + Ok(TableCatalogObjectListPage { + objects: objects.into_iter().collect(), + is_truncated, + }) + } + + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + let lock = self + .store + .new_ns_lock(bucket, object) + .await + .map_err(|err| storage_error_to_catalog("create catalog table lock", err))?; + let guard = lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog table lock: {err}")))?; + Ok(Box::new(guard)) + } + + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + let lock = self + .store + .new_ns_lock(bucket, object) + .await + .map_err(|err| storage_error_to_catalog("create catalog migration lock", err))?; + let guard = lock + .get_read_lock(get_lock_acquire_timeout()) + .await + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog migration lock: {err}")))?; + Ok(Box::new(guard)) + } +} + +impl EcStoreTableCatalogObjectBackend +where + S: TableCatalogStorage, +{ + async fn read_object_with_options( + &self, + bucket: &str, + object: &str, + opts: ObjectOptions, + ) -> TableCatalogStoreResult> { + let info = match self.store.get_object_info(bucket, object, &opts).await { + Ok(info) => info, + Err(err) if is_missing_storage_error(&err) => return Ok(None), + Err(err) => return Err(storage_error_to_catalog("read catalog object info", err)), + }; + let mut reader = match self + .store + .get_object_reader(bucket, object, None, HeaderMap::new(), &opts) + .await + { + Ok(reader) => reader, + Err(err) if is_missing_storage_error(&err) => return Ok(None), + Err(err) => return Err(storage_error_to_catalog("read catalog object", err)), + }; + let mut data = Vec::new(); + reader + .stream + .read_to_end(&mut data) + .await + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to read catalog object {bucket}/{object}: {err}")))?; + Ok(Some(TableCatalogObject { + data, + etag: info.etag, + mod_time: info.mod_time, + })) + } + + async fn put_object_with_options( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: TableCatalogPutPrecondition, + no_lock: bool, + ) -> TableCatalogStoreResult<()> { + let mut reader = PutObjReader::from_vec(data); + let opts = ObjectOptions { + http_preconditions: http_preconditions_for_catalog_put(precondition), + no_lock, + ..Default::default() + }; + self.store + .put_object(bucket, object, &mut reader, &opts) + .await + .map(|_| ()) + .map_err(|err| storage_error_to_catalog("write catalog object", err)) + } +} diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs new file mode 100644 index 000000000..4acac2539 --- /dev/null +++ b/rustfs/src/table_catalog/store/object.rs @@ -0,0 +1,3825 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +#[derive(Clone)] +pub(crate) struct ObjectTableCatalogStore { + pub(in crate::table_catalog) backend: B, + pub(in crate::table_catalog) paths: TableCatalogObjectPaths, +} + +impl ObjectTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + pub fn new(backend: B) -> Self { + Self { + backend, + paths: TableCatalogObjectPaths::default(), + } + } + + pub(in crate::table_catalog) fn catalog_bucket(&self) -> &'static str { + RUSTFS_META_BUCKET + } + + async fn list_entry_page( + &self, + prefix: &str, + entry_file: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> + where + T: DeserializeOwned, + { + let cursor = catalog_list_cursor(cursor, OBJECT_CATALOG_LIST_CURSOR_PREFIX)?; + if cursor.is_some_and(|cursor| !cursor.starts_with(prefix)) { + return Err(TableCatalogStoreError::Invalid( + "page cursor does not match this table catalog list operation".to_string(), + )); + } + + let scan_limit = NonZeroUsize::new(TABLE_CATALOG_LIST_MAX_KEYS) + .ok_or_else(|| TableCatalogStoreError::Internal("catalog object scan limit must be positive".to_string()))?; + let mut entries = Vec::with_capacity(limit.get()); + let mut last_entry_path = None; + + let page = self + .backend + .list_objects_page(self.catalog_bucket(), prefix, cursor, scan_limit) + .await?; + let last_scanned_path = page.objects.last().cloned(); + if page.is_truncated && last_scanned_path.is_none() { + return Err(TableCatalogStoreError::Internal("catalog object pagination made no progress".to_string())); + } + if cursor + .zip(last_scanned_path.as_deref()) + .is_some_and(|(cursor, last)| last <= cursor) + { + return Err(TableCatalogStoreError::Internal("catalog object pagination did not advance".to_string())); + } + + for object in page.objects { + if !object.ends_with(entry_file) { + continue; + } + if entries.len() == limit.get() { + let last_entry_path = last_entry_path.ok_or_else(|| { + TableCatalogStoreError::Internal("catalog page cursor is missing its last entry".to_string()) + })?; + return Ok(TableCatalogListPage { + entries, + next_cursor: Some(format!("{OBJECT_CATALOG_LIST_CURSOR_PREFIX}{last_entry_path}")), + }); + } + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { + continue; + }; + last_entry_path = Some(object); + entries.push(entry); + } + + let next_cursor = if page.is_truncated { + last_scanned_path.map(|path| format!("{OBJECT_CATALOG_LIST_CURSOR_PREFIX}{path}")) + } else { + None + }; + Ok(TableCatalogListPage { entries, next_cursor }) + } + + pub(in crate::table_catalog) async fn read_entry( + &self, + bucket: &str, + object: &str, + ) -> TableCatalogStoreResult)>> + where + T: DeserializeOwned, + { + self.read_entry_with(bucket, object, |backend, bucket, object| { + Box::pin(async move { backend.read_object(bucket, object).await }) + }) + .await + } + + pub(super) async fn read_entry_unlocked( + &self, + bucket: &str, + object: &str, + ) -> TableCatalogStoreResult)>> + where + T: DeserializeOwned, + { + self.read_entry_with(bucket, object, |backend, bucket, object| { + Box::pin(async move { backend.read_object_unlocked(bucket, object).await }) + }) + .await + } + + async fn read_entry_with<'a, T, F>( + &'a self, + bucket: &'a str, + object: &'a str, + read_object: F, + ) -> TableCatalogStoreResult)>> + where + T: DeserializeOwned, + F: FnOnce( + &'a B, + &'a str, + &'a str, + ) -> std::pin::Pin< + Box>> + Send + 'a>, + >, + { + let Some(object_data) = read_object(&self.backend, bucket, object).await? else { + return Ok(None); + }; + + let entry = serde_json::from_slice(&object_data.data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to parse catalog entry {object}: {err}")))?; + Ok(Some((entry, object_data.etag))) + } + + pub(in crate::table_catalog) async fn write_entry( + &self, + bucket: &str, + object: &str, + entry: &T, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> + where + T: Serialize, + { + let data = serde_json::to_vec(entry) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize catalog entry {object}: {err}")))?; + self.backend.put_object(bucket, object, data, precondition).await + } + + async fn write_entry_unlocked( + &self, + bucket: &str, + object: &str, + entry: &T, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> + where + T: Serialize, + { + let data = serde_json::to_vec(entry) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize catalog entry {object}: {err}")))?; + self.backend.put_object_unlocked(bucket, object, data, precondition).await + } + + async fn write_warehouse_index_state_unlocked(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { + let state = TableWarehouseIndexStateEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: table_bucket.to_string(), + state: TableCatalogEntryState::Active, + }; + self.write_entry_unlocked( + self.catalog_bucket(), + &self.paths.warehouse_index_state_path(table_bucket), + &state, + TableCatalogPutPrecondition::Any, + ) + .await + } + + pub(in crate::table_catalog) async fn warehouse_index_ready(&self, table_bucket: &str) -> TableCatalogStoreResult { + let Some((state, _)) = self + .read_entry::( + self.catalog_bucket(), + &self.paths.warehouse_index_state_path(table_bucket), + ) + .await? + else { + return Ok(false); + }; + Ok(state.version == TABLE_CATALOG_ENTRY_VERSION + && state.table_bucket == table_bucket + && state.state == TableCatalogEntryState::Active) + } + + async fn warehouse_index_entry_has_active_owner(&self, index: &TableWarehouseIndexEntry) -> TableCatalogStoreResult { + if index.state != TableCatalogEntryState::Active { + return Ok(false); + } + let Some(table) = self.load_table(&index.table_bucket, &index.namespace, &index.table).await? else { + return Ok(false); + }; + if table.state != TableCatalogEntryState::Active { + return Ok(false); + } + let current_prefix = table_warehouse_object_prefix(&table)?; + Ok(current_prefix == index.warehouse_object_prefix) + } + + async fn delete_warehouse_index_object( + &self, + object: &str, + index: &TableWarehouseIndexEntry, + reason: &'static str, + ) -> TableCatalogStoreResult { + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), object).await?; + let Some((current, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), object) + .await? + else { + return Ok(false); + }; + if current != *index { + tracing::warn!( + table_bucket = %index.table_bucket, + namespace = %index.namespace, + table = %index.table, + table_id = %index.table_id, + warehouse_object_prefix = %index.warehouse_object_prefix, + current_namespace = %current.namespace, + current_table = %current.table, + current_table_id = %current.table_id, + reason = %reason, + "skipped deleting table warehouse index because owner changed" + ); + return Ok(false); + } + self.delete_warehouse_index_object_unlocked(object, index, reason).await?; + Ok(true) + } + + async fn delete_warehouse_index_object_unlocked( + &self, + object: &str, + index: &TableWarehouseIndexEntry, + reason: &'static str, + ) -> TableCatalogStoreResult<()> { + self.backend.delete_object_unlocked(self.catalog_bucket(), object).await?; + tracing::warn!( + table_bucket = %index.table_bucket, + namespace = %index.namespace, + table = %index.table, + table_id = %index.table_id, + warehouse_object_prefix = %index.warehouse_object_prefix, + reason = %reason, + "deleted table warehouse index" + ); + Ok(()) + } + + async fn replace_stale_table_warehouse_index( + &self, + object: &str, + stale: &TableWarehouseIndexEntry, + replacement: &TableWarehouseIndexEntry, + reason: &'static str, + ) -> TableCatalogStoreResult { + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), object).await?; + let Some((current, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), object) + .await? + else { + return Ok(false); + }; + if current != *stale { + return Ok(false); + } + self.delete_warehouse_index_object_unlocked(object, stale, reason).await?; + self.write_entry_unlocked(self.catalog_bucket(), object, replacement, TableCatalogPutPrecondition::IfAbsent) + .await?; + Ok(true) + } + + async fn reserve_table_warehouse_index(&self, entry: &TableEntry) -> TableCatalogStoreResult { + let index = table_warehouse_index_entry(entry)?; + let object = self + .paths + .warehouse_index_entry_path(&index.table_bucket, &index.warehouse_object_prefix); + loop { + match self + .write_entry(self.catalog_bucket(), &object, &index, TableCatalogPutPrecondition::IfAbsent) + .await + { + Ok(()) => return Ok(WarehouseIndexReservation::Created), + Err(TableCatalogStoreError::Conflict(_)) => { + let Some((existing, _)) = self + .read_entry::(self.catalog_bucket(), &object) + .await? + else { + continue; + }; + if existing == index { + return Ok(WarehouseIndexReservation::AlreadyReserved); + } + if existing.table_bucket != index.table_bucket + || existing.warehouse_object_prefix != index.warehouse_object_prefix + || self.warehouse_index_entry_has_active_owner(&existing).await? + { + return Err(TableCatalogStoreError::Conflict(format!( + "table warehouse location is already registered: {}", + index.warehouse_object_prefix + ))); + } + if self + .replace_stale_table_warehouse_index(&object, &existing, &index, "stale reservation conflict") + .await? + { + return Ok(WarehouseIndexReservation::Created); + } + } + Err(err) => return Err(err), + } + } + } + + async fn delete_stale_table_warehouse_index( + &self, + object: &str, + index: &TableWarehouseIndexEntry, + reason: &'static str, + ) -> TableCatalogStoreResult<()> { + self.delete_warehouse_index_object(object, index, reason) + .await + .map(|_| ()) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to delete stale warehouse index {object}: {err}"))) + } + + async fn fail_closed_for_broken_warehouse_index( + &self, + object: &str, + index: &TableWarehouseIndexEntry, + reason: &'static str, + ) -> TableCatalogStoreResult> { + Err(TableCatalogStoreError::Internal(format!( + "active warehouse index {object} for {}/{}/{} ({}) is inconsistent: {reason}", + index.table_bucket, index.namespace, index.table, index.table_id + ))) + } + + async fn resolve_table_data_plane_resource_from_index_entry( + &self, + index_object: &str, + index: TableWarehouseIndexEntry, + ) -> TableCatalogStoreResult> { + if index.state != TableCatalogEntryState::Active { + return Ok(None); + } + let Some(table) = self.load_table(&index.table_bucket, &index.namespace, &index.table).await? else { + return self + .fail_closed_for_broken_warehouse_index(index_object, &index, "referenced table entry is missing") + .await; + }; + if table.state != TableCatalogEntryState::Active { + self.delete_stale_table_warehouse_index(index_object, &index, "referenced table is inactive") + .await?; + return Ok(None); + } + let current_prefix = table_warehouse_object_prefix(&table).map_err(|err| { + TableCatalogStoreError::Invalid(format!("warehouse index table entry has invalid location {index_object}: {err}")) + })?; + if current_prefix != index.warehouse_object_prefix { + self.delete_stale_table_warehouse_index(index_object, &index, "referenced table moved warehouse prefix") + .await?; + return Ok(None); + } + if table.table_id != index.table_id { + return self + .fail_closed_for_broken_warehouse_index(index_object, &index, "referenced table identity changed") + .await; + } + Ok(Some(table_data_plane_resource_from_entry(table, current_prefix))) + } + + async fn read_warehouse_index_state_unlocked(&self, table_bucket: &str) -> TableCatalogStoreResult { + let Some((state, _)) = self + .read_entry_unlocked::( + self.catalog_bucket(), + &self.paths.warehouse_index_state_path(table_bucket), + ) + .await? + else { + return Ok(false); + }; + Ok(state.version == TABLE_CATALOG_ENTRY_VERSION + && state.table_bucket == table_bucket + && state.state == TableCatalogEntryState::Active) + } + + async fn delete_created_table_warehouse_index( + &self, + entry: &TableEntry, + reservation: WarehouseIndexReservation, + reason: &'static str, + ) { + if reservation != WarehouseIndexReservation::Created { + return; + } + let warehouse_object_prefix = table_warehouse_object_prefix(entry).ok(); + if let Err(err) = self.delete_table_warehouse_index(entry).await { + tracing::warn!( + table_bucket = %entry.table_bucket, + namespace = %entry.namespace, + table = %entry.table, + table_id = %entry.table_id, + warehouse_object_prefix = warehouse_object_prefix.as_deref().unwrap_or(""), + reason = %reason, + error = %err, + "failed to roll back table warehouse index reservation" + ); + } + } + + pub(in crate::table_catalog) async fn delete_table_warehouse_index(&self, entry: &TableEntry) -> TableCatalogStoreResult<()> { + let Ok(index) = table_warehouse_index_entry(entry) else { + return Ok(()); + }; + self.delete_warehouse_index_object( + &self + .paths + .warehouse_index_entry_path(&index.table_bucket, &index.warehouse_object_prefix), + &index, + "table warehouse index owner removed", + ) + .await + .map(|_| ()) + } + + async fn delete_table_warehouse_index_if_changed(&self, current: &TableEntry, next: &TableEntry) { + let Ok(current_index) = table_warehouse_index_entry(current) else { + return; + }; + let Ok(next_index) = table_warehouse_index_entry(next) else { + return; + }; + if current_index.warehouse_object_prefix == next_index.warehouse_object_prefix { + return; + } + if let Err(err) = self.delete_table_warehouse_index(current).await { + tracing::warn!( + table_bucket = %current.table_bucket, + namespace = %current.namespace, + table = %current.table, + table_id = %current.table_id, + warehouse_object_prefix = %current_index.warehouse_object_prefix, + error = %err, + "failed to delete stale table warehouse index" + ); + } + } + + async fn resolve_table_data_plane_resource_from_index( + &self, + table_bucket: &str, + object: &str, + ) -> TableCatalogStoreResult> { + for warehouse_object_prefix in warehouse_index_candidate_prefixes(object) { + let index_object = self.paths.warehouse_index_entry_path(table_bucket, warehouse_object_prefix); + let Some((index, _)) = self + .read_entry::(self.catalog_bucket(), &index_object) + .await? + else { + continue; + }; + if index.table_bucket != table_bucket || index.warehouse_object_prefix != warehouse_object_prefix { + return Err(TableCatalogStoreError::Invalid(format!( + "warehouse index entry does not match indexed prefix: {index_object}" + ))); + } + if let Some(resource) = self + .resolve_table_data_plane_resource_from_index_entry(&index_object, index) + .await? + { + return Ok(Some(resource)); + } + } + Ok(None) + } + + pub(in crate::table_catalog) async fn backfill_active_table_warehouse_index( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult<()> { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((current, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Ok(()); + }; + if current.state != TableCatalogEntryState::Active { + return Ok(()); + } + self.reserve_table_warehouse_index(¤t).await.map(|_| ()) + } + + pub(in crate::table_catalog) async fn backfill_table_warehouse_index( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult<()> { + let state_object = self.paths.warehouse_index_state_path(table_bucket); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &state_object).await?; + if self.read_warehouse_index_state_unlocked(table_bucket).await? { + return Ok(()); + } + for namespace in self.list_namespaces(table_bucket).await? { + if namespace.state != TableCatalogEntryState::Active { + continue; + } + for table in self.list_tables(table_bucket, &namespace.namespace).await? { + if table.state != TableCatalogEntryState::Active { + continue; + } + if let Err(err) = self + .backfill_active_table_warehouse_index(&table.table_bucket, &table.namespace, &table.table) + .await + { + if matches!(&err, TableCatalogStoreError::Invalid(_)) { + tracing::warn!( + table_bucket = %table.table_bucket, + namespace = %table.namespace, + table = %table.table, + table_id = %table.table_id, + error = %err, + "skipping invalid table warehouse location while backfilling warehouse index" + ); + continue; + } + return Err(err); + } + } + } + self.write_warehouse_index_state_unlocked(table_bucket).await + } + + async fn require_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { + if self.get_table_bucket(table_bucket).await?.is_none() { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + } + Ok(()) + } + + async fn read_table_with_etag( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + ) -> TableCatalogStoreResult> { + let table_path = self.paths.table_entry_path(table_bucket, namespace, table); + let Some((entry, etag)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Ok(None); + }; + let Some(etag) = etag else { + return Err(TableCatalogStoreError::Internal(format!("catalog table entry has no etag: {table_path}"))); + }; + Ok(Some((entry, etag))) + } + + async fn read_table_with_etag_unlocked( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + ) -> TableCatalogStoreResult> { + let table_path = self.paths.table_entry_path(table_bucket, namespace, table); + let Some((entry, etag)) = self + .read_entry_unlocked::(self.catalog_bucket(), &table_path) + .await? + else { + return Ok(None); + }; + let Some(etag) = etag else { + return Err(TableCatalogStoreError::Internal(format!("catalog table entry has no etag: {table_path}"))); + }; + Ok(Some((entry, etag))) + } + + async fn read_view_with_etag_unlocked( + &self, + table_bucket: &str, + namespace: &Namespace, + view: &IdentifierSegment, + ) -> TableCatalogStoreResult> { + let view_path = self.paths.view_entry_path(table_bucket, namespace, view); + let Some((entry, etag)) = self + .read_entry_unlocked::(self.catalog_bucket(), &view_path) + .await? + else { + return Ok(None); + }; + let Some(etag) = etag else { + return Err(TableCatalogStoreError::Internal(format!("catalog view entry has no etag: {view_path}"))); + }; + Ok(Some((entry, etag))) + } + + async fn write_table_entry( + &self, + entry: TableEntry, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("table", entry.version)?; + self.require_table_bucket(&entry.table_bucket).await?; + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + validate_table_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; + let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); + let _namespace_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &namespace_path) + .await?; + if self + .read_entry_unlocked::(self.catalog_bucket(), &namespace_path) + .await? + .is_none() + { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + let table_path = self.paths.table_entry_path(&entry.table_bucket, &namespace, &table); + let _table_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let reservation = self.reserve_table_warehouse_index(&entry).await?; + let result = self + .write_entry_unlocked(self.catalog_bucket(), &table_path, &entry, precondition) + .await; + if result.is_err() { + self.delete_created_table_warehouse_index(&entry, reservation, "table entry write failed") + .await; + } + result + } + + async fn write_view_entry(&self, entry: ViewEntry, precondition: TableCatalogPutPrecondition) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("view", entry.version)?; + self.require_table_bucket(&entry.table_bucket).await?; + let namespace = parse_namespace_for_store(&entry.namespace)?; + let view = parse_table_for_store(&entry.view)?; + validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; + let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); + let _namespace_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &namespace_path) + .await?; + if self + .read_entry_unlocked::(self.catalog_bucket(), &namespace_path) + .await? + .is_none() + { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + let view_path = self.paths.view_entry_path(&entry.table_bucket, &namespace, &view); + let _view_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &view_path).await?; + self.write_entry_unlocked(self.catalog_bucket(), &view_path, &entry, precondition) + .await + } + + pub(crate) async fn get_external_catalog_bridge( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult> { + self.require_table_bucket(table_bucket).await?; + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + if self.get_namespace(table_bucket, &namespace.public_name()).await?.is_none() { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + table_bucket, + namespace.public_name() + ))); + } + let bridge_path = self.paths.external_catalog_bridge_path(table_bucket, &namespace, &table); + self.read_entry::(self.catalog_bucket(), &bridge_path) + .await + .map(|entry| entry.map(|(bridge, _)| bridge)) + } + + pub(crate) async fn put_external_catalog_bridge( + &self, + entry: ExternalCatalogBridgeEntry, + ) -> TableCatalogStoreResult { + validate_catalog_entry_version("external catalog bridge", entry.version)?; + self.require_table_bucket(&entry.table_bucket).await?; + self.ensure_object_backed_writes_allowed(&entry.table_bucket).await?; + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + let bridge_path = self + .paths + .external_catalog_bridge_path(&entry.table_bucket, &namespace, &table); + self.write_entry(self.catalog_bucket(), &bridge_path, &entry, TableCatalogPutPrecondition::Any) + .await?; + Ok(entry) + } + + async fn read_commit_by_path(&self, object: &str) -> TableCatalogStoreResult> { + self.read_entry::(self.catalog_bucket(), object) + .await + .map(|entry| entry.map(|(commit, _)| commit)) + } + + async fn finalize_commit_log( + &self, + commit_path: &str, + idempotency_path: Option<&str>, + commit_log: &CommitLogEntry, + ) -> TableCatalogStoreResult<()> { + self.write_entry(self.catalog_bucket(), commit_path, commit_log, TableCatalogPutPrecondition::Any) + .await?; + if let Some(idempotency_path) = idempotency_path { + self.write_entry(self.catalog_bucket(), idempotency_path, commit_log, TableCatalogPutPrecondition::Any) + .await?; + } + Ok(()) + } + + pub(super) async fn table_commit_recovery_report_for_entry( + &self, + entry: &TableEntry, + finalized_count: usize, + ) -> TableCatalogStoreResult { + let commit_prefix = self.paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id); + let mut commits = Vec::new(); + for object in self.backend.list_objects(self.catalog_bucket(), &commit_prefix).await? { + if !object.ends_with(".json") { + continue; + } + let Some(commit_log) = self.read_commit_by_path(&object).await? else { + continue; + }; + let idempotency_commit = match commit_log.idempotency_key.as_deref() { + Some(idempotency_key) => { + let idempotency_path = + self.paths + .commit_idempotency_entry_path(&entry.table_bucket, &entry.table_id, idempotency_key); + self.read_commit_by_path(&idempotency_path).await? + } + None => None, + }; + commits.push(table_commit_recovery_entry(entry, &commit_log, idempotency_commit.as_ref())); + } + commits.sort_by(|left, right| left.commit_id.cmp(&right.commit_id)); + + let finalization_required_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::FinalizationRequired)) + .count(); + let idempotency_repair_required_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::IdempotencyIndexRepairRequired)) + .count(); + let staged_before_table_update_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::StagedBeforeTableUpdate)) + .count(); + let manual_review_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::ManualReview)) + .count(); + + Ok(TableCommitRecoveryReport { + table_bucket: entry.table_bucket.clone(), + namespace: entry.namespace.clone(), + table: entry.table.clone(), + table_id: entry.table_id.clone(), + current_metadata_location: entry.metadata_location.clone(), + current_version_token: entry.version_token.clone(), + current_generation: entry.generation, + commits, + staged_before_table_update_count, + finalization_required_count, + idempotency_repair_required_count, + manual_review_count, + finalized_count, + }) + } + + pub(crate) async fn plan_table_commit_recovery( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + self.table_commit_recovery_report_for_entry(&entry, 0).await + } + + pub(crate) async fn recover_table_commits( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + + let commit_prefix = self.paths.commit_log_entries_prefix(table_bucket, &entry.table_id); + let mut finalized_count = 0; + for commit_path in self.backend.list_objects(self.catalog_bucket(), &commit_prefix).await? { + if !commit_path.ends_with(".json") { + continue; + } + let Some(commit_log) = self.read_commit_by_path(&commit_path).await? else { + continue; + }; + let idempotency_path = commit_log.idempotency_key.as_deref().map(|idempotency_key| { + self.paths + .commit_idempotency_entry_path(table_bucket, &entry.table_id, idempotency_key) + }); + let idempotency_commit = match idempotency_path.as_deref() { + Some(idempotency_path) => self.read_commit_by_path(idempotency_path).await?, + None => None, + }; + let recovery_entry = table_commit_recovery_entry(&entry, &commit_log, idempotency_commit.as_ref()); + if matches!( + recovery_entry.recovery_state, + TableCommitRecoveryState::FinalizationRequired | TableCommitRecoveryState::IdempotencyIndexRepairRequired + ) { + let mut committed = commit_log; + committed.status = CommitLogStatus::Committed; + self.finalize_commit_log(&commit_path, idempotency_path.as_deref(), &committed) + .await?; + finalized_count += 1; + } + } + + self.table_commit_recovery_report_for_entry(&entry, finalized_count).await + } + + pub(crate) async fn get_table_maintenance_config( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + + let config_path = self + .paths + .table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id); + let config = self + .read_entry::(self.catalog_bucket(), &config_path) + .await? + .map(|(config, _)| config) + .unwrap_or_default(); + validate_table_maintenance_config(&config)?; + Ok(config) + } + + pub(crate) async fn put_table_bucket_maintenance_config( + &self, + table_bucket: &str, + config: TableMaintenanceConfig, + ) -> TableCatalogStoreResult { + validate_table_maintenance_config(&config)?; + self.require_table_bucket(table_bucket).await?; + self.ensure_object_backed_writes_allowed(table_bucket).await?; + let config_path = self.paths.table_bucket_maintenance_config_path(table_bucket); + self.write_entry(self.catalog_bucket(), &config_path, &config, TableCatalogPutPrecondition::Any) + .await?; + Ok(config) + } + + pub(crate) async fn get_effective_table_maintenance_config( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + + self.get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) + .await + } + + async fn get_effective_table_maintenance_config_for_entry_unlocked( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + entry: &TableEntry, + ) -> TableCatalogStoreResult { + let table_config_path = self + .paths + .table_maintenance_config_path(table_bucket, namespace, table, &entry.table_id); + if let Some((config, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), &table_config_path) + .await? + { + validate_table_maintenance_config(&config)?; + return Ok(TableMaintenanceEffectiveConfig { + config, + source: TableMaintenanceConfigSource::TableOverride, + }); + } + + let bucket_config_path = self.paths.table_bucket_maintenance_config_path(table_bucket); + if let Some((config, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), &bucket_config_path) + .await? + { + validate_table_maintenance_config(&config)?; + return Ok(TableMaintenanceEffectiveConfig { + config, + source: TableMaintenanceConfigSource::TableBucketDefault, + }); + } + + Ok(TableMaintenanceEffectiveConfig { + config: TableMaintenanceConfig::default(), + source: TableMaintenanceConfigSource::Default, + }) + } + + pub(crate) async fn put_table_maintenance_config( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + config: TableMaintenanceConfig, + ) -> TableCatalogStoreResult { + validate_table_maintenance_config(&config)?; + self.ensure_object_backed_writes_allowed(table_bucket).await?; + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + + let config_path = self + .paths + .table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id); + self.write_entry(self.catalog_bucket(), &config_path, &config, TableCatalogPutPrecondition::Any) + .await?; + Ok(config) + } + + pub(crate) async fn put_table_metadata_maintenance_report( + &self, + report: &TableMetadataMaintenanceReport, + ) -> TableCatalogStoreResult<()> { + let report = table_maintenance_report_with_recommended_actions(report.clone()); + self.ensure_object_backed_writes_allowed(&report.job.table_bucket).await?; + let namespace = parse_namespace_for_store(&report.job.namespace)?; + let table = parse_table_for_store(&report.job.table)?; + let job_path = self.paths.table_maintenance_job_path( + &report.job.table_bucket, + &namespace, + &table, + &report.job.table_id, + &report.job.job_id, + ); + let latest_job_path = + self.paths + .table_maintenance_latest_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id); + let current_job_path = + self.paths + .table_maintenance_current_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id); + self.write_entry(self.catalog_bucket(), &job_path, &report, TableCatalogPutPrecondition::Any) + .await?; + self.write_entry(self.catalog_bucket(), &latest_job_path, &report, TableCatalogPutPrecondition::Any) + .await?; + self.write_entry(self.catalog_bucket(), ¤t_job_path, &report, TableCatalogPutPrecondition::Any) + .await + } + + pub(crate) async fn get_table_metadata_maintenance_report( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + job_id: &str, + ) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + + self.get_table_metadata_maintenance_report_for_entry_unlocked(table_bucket, &namespace, &table, &entry.table_id, job_id) + .await + } + + async fn get_table_metadata_maintenance_report_for_entry_unlocked( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, + job_id: &str, + ) -> TableCatalogStoreResult> { + let job_path = self.table_metadata_maintenance_report_path(table_bucket, namespace, table, table_id, job_id); + self.read_entry_unlocked::(self.catalog_bucket(), &job_path) + .await + .map(|entry| entry.map(|(report, _)| table_maintenance_report_with_recommended_actions(report))) + } + + fn table_metadata_maintenance_report_path( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, + job_id: &str, + ) -> String { + match job_id { + MAINTENANCE_JOB_ALIAS_LATEST => { + self.paths + .table_maintenance_latest_job_path(table_bucket, namespace, table, table_id) + } + MAINTENANCE_JOB_ALIAS_CURRENT => { + self.paths + .table_maintenance_current_job_path(table_bucket, namespace, table, table_id) + } + _ => self + .paths + .table_maintenance_job_path(table_bucket, namespace, table, table_id, job_id), + } + } + + pub(crate) async fn get_table_maintenance_scheduler_report( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + self.get_table_maintenance_scheduler_report_at(table_bucket, namespace, table, OffsetDateTime::now_utc()) + .await + } + + pub(in crate::table_catalog) async fn get_table_maintenance_scheduler_report_at( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + now: OffsetDateTime, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + let effective = self + .get_effective_table_maintenance_config(table_bucket, &namespace.public_name(), table.as_str()) + .await?; + let current = self + .get_table_metadata_maintenance_report( + table_bucket, + &namespace.public_name(), + table.as_str(), + MAINTENANCE_JOB_ALIAS_CURRENT, + ) + .await?; + let reports = self + .list_table_metadata_maintenance_audit_reports(table_bucket, &namespace, &table, &entry.table_id) + .await?; + let quarantine = table_maintenance_scheduler_quarantine_boundary(&effective.config, &reports); + let mut recommended_actions = Vec::new(); + + let status = if !effective.config.background_enabled { + push_unique_maintenance_action( + &mut recommended_actions, + TableMaintenanceRecommendedAction::EnableBackgroundMaintenance, + ); + TableMaintenanceSchedulerStatus::Disabled + } else if effective.config.worker_paused { + push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::ResumeMaintenanceWorker); + TableMaintenanceSchedulerStatus::Paused + } else if let Some(current) = current.as_ref() + && matches!(current.job.status, TableMetadataMaintenanceJobStatus::Running) + && table_maintenance_job_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) + { + push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::WaitForActiveWorker); + TableMaintenanceSchedulerStatus::Backpressured + } else if let Some(current) = current.as_ref() + && matches!(current.job.status, TableMetadataMaintenanceJobStatus::Queued) + && table_maintenance_scheduler_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) + { + push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::RunMaintenanceWorker); + TableMaintenanceSchedulerStatus::Queued + } else if let Some(current) = current.as_ref() + && table_maintenance_job_retry_is_pending(¤t.job, now) + { + push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::WaitForRetryBackoff); + TableMaintenanceSchedulerStatus::RetryDeferred + } else if quarantine.active { + push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::ReviewQuarantine); + push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::InvestigateFailure); + TableMaintenanceSchedulerStatus::Quarantined + } else { + push_unique_maintenance_action(&mut recommended_actions, TableMaintenanceRecommendedAction::NoActionRequired); + TableMaintenanceSchedulerStatus::Ready + }; + + Ok(TableMaintenanceSchedulerReport { + table_bucket: table_bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: entry.table_id, + status, + config_source: effective.source, + background_enabled: effective.config.background_enabled, + worker_paused: effective.config.worker_paused, + delete_enabled: effective.config.delete_enabled, + worker_lease_timeout_seconds: effective.config.worker_lease_timeout_seconds, + max_retry_attempts: effective.config.max_retry_attempts, + retry_initial_backoff_seconds: effective.config.retry_initial_backoff_seconds, + retry_max_backoff_seconds: effective.config.retry_max_backoff_seconds, + recommended_actions, + current_job: current.as_ref().map(table_maintenance_scheduler_job_summary), + quarantine, + audit_timeline: reports.iter().map(table_maintenance_scheduler_job_summary).collect(), + }) + } + + pub(crate) async fn run_table_maintenance_scheduler_once( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + scheduler_id: String, + ) -> TableCatalogStoreResult { + self.run_table_maintenance_scheduler_once_at(table_bucket, namespace, table, scheduler_id, OffsetDateTime::now_utc()) + .await + } + + pub(in crate::table_catalog) async fn run_table_maintenance_scheduler_once_at( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + scheduler_id: String, + now: OffsetDateTime, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let namespace_name = namespace.public_name(); + let table_name = table.as_str().to_string(); + + let preflight = { + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, namespace_name, table_name + ))); + }; + let effective = self + .get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) + .await?; + self.table_metadata_maintenance_scheduler_preflight( + TableMaintenancePreflightContext { + table_bucket, + namespace: &namespace, + table: &table, + entry: &entry, + }, + &scheduler_id, + now, + effective, + ) + .await? + }; + let effective = match preflight { + TableMaintenanceSchedulerPreflight::Ready(effective) => effective, + TableMaintenanceSchedulerPreflight::Complete(report) => { + let scheduler = self + .get_table_maintenance_scheduler_report_at(table_bucket, &namespace_name, &table_name, now) + .await?; + return Ok(TableMaintenanceSchedulerRunResult { + report: *report, + scheduler, + }); + } + }; + + let mut report = self + .plan_table_metadata_maintenance( + table_bucket, + &namespace_name, + &table_name, + effective.config.retain_recent_metadata_files, + ) + .await?; + + let report = { + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, namespace_name, table_name + ))); + }; + let effective = self + .get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) + .await?; + match self + .table_metadata_maintenance_scheduler_preflight( + TableMaintenancePreflightContext { + table_bucket, + namespace: &namespace, + table: &table, + entry: &entry, + }, + &scheduler_id, + now, + effective, + ) + .await? + { + TableMaintenanceSchedulerPreflight::Ready(effective) => { + if report.job.retain_recent_metadata_files != effective.config.retain_recent_metadata_files { + return Err(TableCatalogStoreError::Conflict( + "maintenance config changed before scheduler claim".to_string(), + )); + } + if entry.metadata_location != report.current_metadata_location { + return Err(TableCatalogStoreError::Conflict( + "current metadata location changed before maintenance scheduler claim".to_string(), + )); + } + + let before_status = Some(report.job.status.clone()); + let before_quarantined_object_count = Some(report.job.quarantined_object_count); + let scheduled_at = maintenance_timestamp(now); + report.job.operation = if effective.config.delete_enabled { + TableMetadataMaintenanceOperation::Delete + } else { + TableMetadataMaintenanceOperation::DryRun + }; + report.job.status = TableMetadataMaintenanceJobStatus::Queued; + report.job.failure_reason = None; + report.job.config_source = effective.source; + report.job.scheduler_id = Some(scheduler_id); + report.job.scheduler_lease_id = Uuid::new_v4().to_string(); + report.job.scheduled_at = Some(scheduled_at); + report.job.worker_id = None; + report.job.lease_id = String::new(); + report.job.attempt = 0; + report.job.max_retry_attempts = effective.config.max_retry_attempts; + report.job.next_retry_after = None; + report.job.quarantine_enabled = effective.config.quarantine_enabled; + report.job.quarantine_retention_seconds = effective.config.quarantine_retention_seconds; + report.job.heartbeat_at = None; + report.job.started_at = None; + report.job.finished_at = None; + refresh_table_maintenance_report_recommended_actions(&mut report); + push_table_maintenance_audit_event( + &mut report, + now, + TableMaintenanceAuditActor::Scheduler, + TableMaintenanceAuditAction::SchedulerQueued, + None, + before_status, + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&report).await?; + report + } + TableMaintenanceSchedulerPreflight::Complete(report) => *report, + } + }; + + let scheduler = self + .get_table_maintenance_scheduler_report_at(table_bucket, &namespace_name, &table_name, now) + .await?; + Ok(TableMaintenanceSchedulerRunResult { report, scheduler }) + } + + pub(crate) async fn apply_table_maintenance_quarantine_operation( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + job_id: &str, + request: TableMaintenanceQuarantineOperationRequest, + ) -> TableCatalogStoreResult { + let action = request.action.clone(); + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let namespace_name = namespace.public_name(); + let table_name = table.as_str().to_string(); + + let report = if matches!(action, TableMaintenanceQuarantineAction::Inspect) { + self.get_table_metadata_maintenance_report(table_bucket, &namespace_name, &table_name, job_id) + .await? + .ok_or_else(|| { + TableCatalogStoreError::NotFound(format!( + "maintenance job {}/{}/{}/{}", + table_bucket, namespace_name, table_name, job_id + )) + })? + } else { + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, namespace_name, table_name + ))); + }; + let Some(mut report) = self + .get_table_metadata_maintenance_report_for_entry_unlocked( + table_bucket, + &namespace, + &table, + &entry.table_id, + MAINTENANCE_JOB_ALIAS_CURRENT, + ) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "maintenance job {}/{}/{}/{}", + table_bucket, namespace_name, table_name, job_id + ))); + }; + if report.job.job_id != job_id { + return Err(TableCatalogStoreError::Conflict("maintenance job is not current".to_string())); + } + if !matches!(report.job.status, TableMetadataMaintenanceJobStatus::Failed) { + return Err(TableCatalogStoreError::Conflict( + "maintenance quarantine operation requires a failed job".to_string(), + )); + } + if !report.job.quarantine_enabled || report.job.quarantined_object_count == 0 { + return Err(TableCatalogStoreError::Conflict( + "maintenance job has no active quarantine boundary".to_string(), + )); + } + + let before_status = Some(report.job.status.clone()); + let before_quarantined_object_count = Some(report.job.quarantined_object_count); + report.job.quarantined_object_count = 0; + match &action { + TableMaintenanceQuarantineAction::Inspect => unreachable!("inspect branch handled before mutation"), + TableMaintenanceQuarantineAction::Release => { + report.job.failure_reason = + Some(table_maintenance_quarantine_operator_reason("released", request.reason.as_deref())); + } + TableMaintenanceQuarantineAction::Retry => { + report.job.next_retry_after = None; + report.job.failure_reason = Some(table_maintenance_quarantine_operator_reason( + "released for retry", + request.reason.as_deref(), + )); + } + TableMaintenanceQuarantineAction::Abandon => { + report.job.next_retry_after = None; + report.job.failure_reason = + Some(table_maintenance_quarantine_operator_reason("abandoned", request.reason.as_deref())); + } + } + refresh_table_maintenance_report_recommended_actions(&mut report); + let audit_action = match &action { + TableMaintenanceQuarantineAction::Inspect => unreachable!("inspect branch handled before mutation"), + TableMaintenanceQuarantineAction::Release => TableMaintenanceAuditAction::QuarantineRelease, + TableMaintenanceQuarantineAction::Retry => TableMaintenanceAuditAction::QuarantineRetry, + TableMaintenanceQuarantineAction::Abandon => TableMaintenanceAuditAction::QuarantineAbandon, + }; + push_table_maintenance_audit_event( + &mut report, + OffsetDateTime::now_utc(), + TableMaintenanceAuditActor::Operator, + audit_action, + request.reason, + before_status, + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&report).await?; + report + }; + + let scheduler = self + .get_table_maintenance_scheduler_report(table_bucket, &namespace_name, &table_name) + .await?; + Ok(TableMaintenanceQuarantineOperationResult { + action, + report, + scheduler, + }) + } + + async fn list_table_metadata_maintenance_audit_reports( + &self, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, + ) -> TableCatalogStoreResult> { + let jobs_prefix = self + .paths + .table_maintenance_jobs_prefix(table_bucket, namespace, table, table_id); + let mut reports = Vec::new(); + for object in self.backend.list_objects(self.catalog_bucket(), &jobs_prefix).await? { + if !object.ends_with(".json") { + continue; + } + if let Some((report, _)) = self + .read_entry::(self.catalog_bucket(), &object) + .await? + { + reports.push(table_maintenance_report_with_recommended_actions(report)); + } + } + reports.sort_by(|left, right| { + table_maintenance_report_order_timestamp(right) + .cmp(&table_maintenance_report_order_timestamp(left)) + .then_with(|| left.job.job_id.cmp(&right.job.job_id)) + }); + reports.truncate(TABLE_MAINTENANCE_SCHEDULER_AUDIT_LIMIT); + Ok(reports) + } + + async fn table_metadata_maintenance_scheduler_preflight( + &self, + context: TableMaintenancePreflightContext<'_>, + scheduler_id: &str, + now: OffsetDateTime, + effective: TableMaintenanceEffectiveConfig, + ) -> TableCatalogStoreResult { + if !effective.config.background_enabled { + let report = self + .put_table_metadata_maintenance_scheduler_control_report(TableMaintenanceSchedulerControlReport { + table_bucket: context.table_bucket, + namespace: context.namespace, + table: context.table, + entry: context.entry, + scheduler_id: scheduler_id.to_string(), + effective: &effective, + status: TableMetadataMaintenanceJobStatus::Disabled, + reason: "background maintenance is disabled", + now, + }) + .await?; + return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(report))); + } + if effective.config.worker_paused { + let report = self + .put_table_metadata_maintenance_scheduler_control_report(TableMaintenanceSchedulerControlReport { + table_bucket: context.table_bucket, + namespace: context.namespace, + table: context.table, + entry: context.entry, + scheduler_id: scheduler_id.to_string(), + effective: &effective, + status: TableMetadataMaintenanceJobStatus::Paused, + reason: "background maintenance worker is paused", + now, + }) + .await?; + return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(report))); + } + + if let Some(current) = self + .get_table_metadata_maintenance_report_for_entry_unlocked( + context.table_bucket, + context.namespace, + context.table, + &context.entry.table_id, + MAINTENANCE_JOB_ALIAS_CURRENT, + ) + .await? + { + if matches!(current.job.status, TableMetadataMaintenanceJobStatus::Running) { + if table_maintenance_job_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) { + return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(current))); + } + self.expire_table_maintenance_job( + current, + now, + "maintenance worker lease expired", + TableMaintenanceAuditAction::WorkerLeaseExpired, + ) + .await?; + } else if matches!(current.job.status, TableMetadataMaintenanceJobStatus::Queued) { + if table_maintenance_scheduler_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) { + return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(current))); + } + self.expire_table_maintenance_job( + current, + now, + "maintenance scheduler lease expired", + TableMaintenanceAuditAction::SchedulerLeaseExpired, + ) + .await?; + } else if table_maintenance_job_retry_is_pending(¤t.job, now) { + return Ok(TableMaintenanceSchedulerPreflight::Complete(Box::new(current))); + } + } + + Ok(TableMaintenanceSchedulerPreflight::Ready(effective)) + } + + pub(crate) async fn run_table_metadata_maintenance_worker_once( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + worker_id: String, + ) -> TableCatalogStoreResult { + self.run_table_metadata_maintenance_worker_once_at(table_bucket, namespace, table, worker_id, OffsetDateTime::now_utc()) + .await + } + + pub(in crate::table_catalog) async fn run_table_metadata_maintenance_worker_once_at( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + worker_id: String, + now: OffsetDateTime, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let namespace_name = namespace.public_name(); + let table_name = table.as_str().to_string(); + + let (effective, queued) = { + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, namespace_name, table_name + ))); + }; + let effective = self + .get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) + .await?; + match self + .table_metadata_maintenance_worker_preflight( + TableMaintenancePreflightContext { + table_bucket, + namespace: &namespace, + table: &table, + entry: &entry, + }, + &worker_id, + now, + effective, + ) + .await? + { + TableMaintenanceWorkerPreflight::Ready { effective, queued } => (effective, queued), + TableMaintenanceWorkerPreflight::Complete(report) => return Ok(*report), + } + }; + + let mut report = if let Some(queued) = queued { + *queued + } else { + self.plan_table_metadata_maintenance( + table_bucket, + &namespace_name, + &table_name, + effective.config.retain_recent_metadata_files, + ) + .await? + }; + + let (report, effective, delete) = { + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, namespace_name, table_name + ))); + }; + let effective = self + .get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry) + .await?; + let (effective, queued) = match self + .table_metadata_maintenance_worker_preflight( + TableMaintenancePreflightContext { + table_bucket, + namespace: &namespace, + table: &table, + entry: &entry, + }, + &worker_id, + now, + effective, + ) + .await? + { + TableMaintenanceWorkerPreflight::Ready { effective, queued } => (effective, queued), + TableMaintenanceWorkerPreflight::Complete(report) => return Ok(*report), + }; + if let Some(queued) = queued { + if queued.job.job_id != report.job.job_id { + return Err(TableCatalogStoreError::Conflict( + "queued maintenance job changed before worker claim".to_string(), + )); + } + report = *queued; + } + + if report.job.retain_recent_metadata_files != effective.config.retain_recent_metadata_files { + return Err(TableCatalogStoreError::Conflict( + "maintenance config changed before worker claim".to_string(), + )); + } + if entry.metadata_location != report.current_metadata_location { + return Err(TableCatalogStoreError::Conflict( + "current metadata location changed before maintenance worker claim".to_string(), + )); + } + + let was_queued_claim = matches!(report.job.status, TableMetadataMaintenanceJobStatus::Queued); + let before_status = Some(report.job.status.clone()); + let before_quarantined_object_count = Some(report.job.quarantined_object_count); + let started_at = maintenance_timestamp(now); + if !was_queued_claim { + report.job.operation = if effective.config.delete_enabled { + TableMetadataMaintenanceOperation::Delete + } else { + TableMetadataMaintenanceOperation::DryRun + }; + } + report.job.status = TableMetadataMaintenanceJobStatus::Running; + report.job.failure_reason = None; + report.job.config_source = effective.source; + report.job.worker_id = Some(worker_id); + report.job.lease_id = Uuid::new_v4().to_string(); + report.job.attempt = 1; + report.job.max_retry_attempts = effective.config.max_retry_attempts; + report.job.next_retry_after = None; + report.job.quarantine_enabled = effective.config.quarantine_enabled; + report.job.quarantine_retention_seconds = effective.config.quarantine_retention_seconds; + report.job.heartbeat_at = Some(started_at.clone()); + report.job.started_at = Some(started_at); + report.job.finished_at = None; + refresh_table_maintenance_report_recommended_actions(&mut report); + push_table_maintenance_audit_event( + &mut report, + now, + TableMaintenanceAuditActor::Worker, + TableMaintenanceAuditAction::WorkerStarted, + None, + before_status, + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&report).await?; + + let delete = matches!(report.job.operation, TableMetadataMaintenanceOperation::Delete); + (report, effective, delete) + }; + + self.finish_table_metadata_maintenance_run(table_bucket, &namespace_name, &table_name, delete, &effective, report) + .await + } + + async fn table_metadata_maintenance_worker_preflight( + &self, + context: TableMaintenancePreflightContext<'_>, + worker_id: &str, + now: OffsetDateTime, + effective: TableMaintenanceEffectiveConfig, + ) -> TableCatalogStoreResult { + if !effective.config.background_enabled { + let report = self + .put_table_metadata_maintenance_worker_control_report(TableMaintenanceWorkerControlReport { + table_bucket: context.table_bucket, + namespace: context.namespace, + table: context.table, + entry: context.entry, + worker_id: worker_id.to_string(), + effective: &effective, + status: TableMetadataMaintenanceJobStatus::Disabled, + reason: "background maintenance is disabled", + now, + }) + .await?; + return Ok(TableMaintenanceWorkerPreflight::Complete(Box::new(report))); + } + if effective.config.worker_paused { + let report = self + .put_table_metadata_maintenance_worker_control_report(TableMaintenanceWorkerControlReport { + table_bucket: context.table_bucket, + namespace: context.namespace, + table: context.table, + entry: context.entry, + worker_id: worker_id.to_string(), + effective: &effective, + status: TableMetadataMaintenanceJobStatus::Paused, + reason: "background maintenance worker is paused", + now, + }) + .await?; + return Ok(TableMaintenanceWorkerPreflight::Complete(Box::new(report))); + } + + if let Some(current) = self + .get_table_metadata_maintenance_report_for_entry_unlocked( + context.table_bucket, + context.namespace, + context.table, + &context.entry.table_id, + MAINTENANCE_JOB_ALIAS_CURRENT, + ) + .await? + { + if matches!(current.job.status, TableMetadataMaintenanceJobStatus::Running) { + if table_maintenance_job_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) { + return Ok(TableMaintenanceWorkerPreflight::Complete(Box::new(current))); + } + self.expire_table_maintenance_job( + current, + now, + "maintenance worker lease expired", + TableMaintenanceAuditAction::WorkerLeaseExpired, + ) + .await?; + } else if matches!(current.job.status, TableMetadataMaintenanceJobStatus::Queued) { + if table_maintenance_scheduler_lease_is_active(¤t.job, effective.config.worker_lease_timeout_seconds, now) { + return Ok(TableMaintenanceWorkerPreflight::Ready { + effective, + queued: Some(Box::new(current)), + }); + } + self.expire_table_maintenance_job( + current, + now, + "maintenance scheduler lease expired", + TableMaintenanceAuditAction::SchedulerLeaseExpired, + ) + .await?; + } else if table_maintenance_job_retry_is_pending(¤t.job, now) { + return Ok(TableMaintenanceWorkerPreflight::Complete(Box::new(current))); + } + } + + Ok(TableMaintenanceWorkerPreflight::Ready { effective, queued: None }) + } + + pub(crate) async fn heartbeat_table_metadata_maintenance_job( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + job_id: &str, + lease_id: &str, + worker_id: &str, + ) -> TableCatalogStoreResult { + self.heartbeat_table_metadata_maintenance_job_at( + TableMaintenanceHeartbeatRef { + table_bucket, + namespace, + table, + job_id, + lease_id, + worker_id, + }, + OffsetDateTime::now_utc(), + ) + .await + } + + pub(in crate::table_catalog) async fn heartbeat_table_metadata_maintenance_job_at( + &self, + heartbeat: TableMaintenanceHeartbeatRef<'_>, + now: OffsetDateTime, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(heartbeat.namespace)?; + let table = parse_table_for_store(heartbeat.table)?; + let table_path = self.paths.table_entry_path(heartbeat.table_bucket, &namespace, &table); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((entry, _)) = self + .read_table_with_etag_unlocked(heartbeat.table_bucket, &namespace, &table) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + heartbeat.table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + let Some(mut report) = self + .get_table_metadata_maintenance_report_for_entry_unlocked( + heartbeat.table_bucket, + &namespace, + &table, + &entry.table_id, + MAINTENANCE_JOB_ALIAS_CURRENT, + ) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "maintenance job {}/{}/{}/{}", + heartbeat.table_bucket, + namespace.public_name(), + table.as_str(), + heartbeat.job_id + ))); + }; + if report.job.job_id != heartbeat.job_id { + return Err(TableCatalogStoreError::Conflict("maintenance job is not current".to_string())); + } + if !matches!(report.job.status, TableMetadataMaintenanceJobStatus::Running) { + return Err(TableCatalogStoreError::Conflict("maintenance job is not running".to_string())); + } + if report.job.lease_id != heartbeat.lease_id { + return Err(TableCatalogStoreError::Conflict("maintenance lease does not match".to_string())); + } + if report.job.worker_id.as_deref() != Some(heartbeat.worker_id) { + return Err(TableCatalogStoreError::Conflict("maintenance worker does not match".to_string())); + } + + report.job.heartbeat_at = Some(maintenance_timestamp(now)); + refresh_table_maintenance_report_recommended_actions(&mut report); + let before_quarantined_object_count = Some(report.job.quarantined_object_count); + push_table_maintenance_audit_event( + &mut report, + now, + TableMaintenanceAuditActor::Worker, + TableMaintenanceAuditAction::WorkerHeartbeat, + None, + Some(TableMetadataMaintenanceJobStatus::Running), + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&report).await?; + Ok(report) + } + + async fn expire_table_maintenance_job( + &self, + mut report: TableMetadataMaintenanceReport, + now: OffsetDateTime, + reason: &str, + action: TableMaintenanceAuditAction, + ) -> TableCatalogStoreResult { + let before_status = Some(report.job.status.clone()); + let before_quarantined_object_count = Some(report.job.quarantined_object_count); + report.job.status = TableMetadataMaintenanceJobStatus::Failed; + report.job.failure_reason = Some(reason.to_string()); + report.job.finished_at = Some(maintenance_timestamp(now)); + refresh_table_maintenance_report_recommended_actions(&mut report); + push_table_maintenance_audit_event( + &mut report, + now, + TableMaintenanceAuditActor::Scheduler, + action, + Some(reason.to_string()), + before_status, + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&report).await?; + Ok(report) + } + + async fn put_table_metadata_maintenance_scheduler_control_report( + &self, + control: TableMaintenanceSchedulerControlReport<'_>, + ) -> TableCatalogStoreResult { + let timestamp = maintenance_timestamp(control.now); + let cleanup_watermark_unix_seconds = + (control.now - Duration::seconds(TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS)).unix_timestamp(); + let current_metadata_location = control.entry.metadata_location.clone(); + let report = TableMetadataMaintenanceReport { + job: TableMetadataMaintenanceJob { + job_id: Uuid::new_v4().to_string(), + table_bucket: control.table_bucket.to_string(), + namespace: control.namespace.public_name(), + table: control.table.as_str().to_string(), + table_id: control.entry.table_id.clone(), + operation: TableMetadataMaintenanceOperation::DryRun, + status: control.status, + failure_reason: Some(control.reason.to_string()), + recommended_actions: Vec::new(), + config_source: control.effective.source, + scheduler_id: Some(control.scheduler_id), + scheduler_lease_id: String::new(), + scheduled_at: Some(timestamp.clone()), + worker_id: None, + lease_id: String::new(), + attempt: 0, + max_retry_attempts: control.effective.config.max_retry_attempts, + next_retry_after: None, + quarantine_enabled: control.effective.config.quarantine_enabled, + quarantine_retention_seconds: control.effective.config.quarantine_retention_seconds, + heartbeat_at: None, + started_at: None, + finished_at: Some(timestamp), + current_metadata_location: current_metadata_location.clone(), + current_generation: control.entry.generation, + retain_recent_metadata_files: control.effective.config.retain_recent_metadata_files, + safety_window_seconds: TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS, + cleanup_watermark_unix_seconds, + planned_metadata_file_count: 0, + retained_metadata_file_count: 0, + cleanup_candidate_count: 0, + deletable_metadata_file_count: 0, + deleted_metadata_file_count: 0, + planned_object_file_count: 0, + cleanup_candidate_object_count: 0, + deletable_object_count: 0, + deleted_object_count: 0, + quarantined_object_count: 0, + }, + current_metadata_location, + retained_metadata_locations: Vec::new(), + cleanup_candidate_locations: Vec::new(), + deletable_metadata_locations: Vec::new(), + cleanup_object_candidate_locations: Vec::new(), + deletable_object_locations: Vec::new(), + object_reports: Vec::new(), + object_cleanup_reports: Vec::new(), + referenced_object_reports: Vec::new(), + reachability_graph: TableMaintenanceReachabilityGraphReport::default(), + snapshot_expiration: None, + compaction: None, + audit_events: Vec::new(), + }; + let mut report = table_maintenance_report_with_recommended_actions(report); + push_table_maintenance_audit_event( + &mut report, + control.now, + TableMaintenanceAuditActor::Scheduler, + TableMaintenanceAuditAction::SchedulerControl, + Some(control.reason.to_string()), + None, + None, + ); + self.put_table_metadata_maintenance_report(&report).await?; + Ok(report) + } + + async fn put_table_metadata_maintenance_worker_control_report( + &self, + control: TableMaintenanceWorkerControlReport<'_>, + ) -> TableCatalogStoreResult { + let timestamp = maintenance_timestamp(control.now); + let cleanup_watermark_unix_seconds = + (control.now - Duration::seconds(TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS)).unix_timestamp(); + let current_metadata_location = control.entry.metadata_location.clone(); + let report = TableMetadataMaintenanceReport { + job: TableMetadataMaintenanceJob { + job_id: Uuid::new_v4().to_string(), + table_bucket: control.table_bucket.to_string(), + namespace: control.namespace.public_name(), + table: control.table.as_str().to_string(), + table_id: control.entry.table_id.clone(), + operation: TableMetadataMaintenanceOperation::DryRun, + status: control.status, + failure_reason: Some(control.reason.to_string()), + recommended_actions: Vec::new(), + config_source: control.effective.source, + scheduler_id: None, + scheduler_lease_id: String::new(), + scheduled_at: None, + worker_id: Some(control.worker_id), + lease_id: String::new(), + attempt: 0, + max_retry_attempts: control.effective.config.max_retry_attempts, + next_retry_after: None, + quarantine_enabled: control.effective.config.quarantine_enabled, + quarantine_retention_seconds: control.effective.config.quarantine_retention_seconds, + heartbeat_at: None, + started_at: Some(timestamp.clone()), + finished_at: Some(timestamp), + current_metadata_location: current_metadata_location.clone(), + current_generation: control.entry.generation, + retain_recent_metadata_files: control.effective.config.retain_recent_metadata_files, + safety_window_seconds: TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS, + cleanup_watermark_unix_seconds, + planned_metadata_file_count: 0, + retained_metadata_file_count: 0, + cleanup_candidate_count: 0, + deletable_metadata_file_count: 0, + deleted_metadata_file_count: 0, + planned_object_file_count: 0, + cleanup_candidate_object_count: 0, + deletable_object_count: 0, + deleted_object_count: 0, + quarantined_object_count: 0, + }, + current_metadata_location, + retained_metadata_locations: Vec::new(), + cleanup_candidate_locations: Vec::new(), + deletable_metadata_locations: Vec::new(), + cleanup_object_candidate_locations: Vec::new(), + deletable_object_locations: Vec::new(), + object_reports: Vec::new(), + object_cleanup_reports: Vec::new(), + referenced_object_reports: Vec::new(), + reachability_graph: TableMaintenanceReachabilityGraphReport::default(), + snapshot_expiration: None, + compaction: None, + audit_events: Vec::new(), + }; + let mut report = table_maintenance_report_with_recommended_actions(report); + push_table_maintenance_audit_event( + &mut report, + control.now, + TableMaintenanceAuditActor::Scheduler, + TableMaintenanceAuditAction::WorkerControl, + Some(control.reason.to_string()), + None, + None, + ); + self.put_table_metadata_maintenance_report(&report).await?; + Ok(report) + } + + pub(crate) async fn plan_table_snapshot_expiration( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + config: TableSnapshotExpirationConfig, + ) -> TableCatalogStoreResult { + validate_table_snapshot_expiration_config(&config)?; + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "current metadata location must be inside the table metadata directory".to_string(), + )); + } + + let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "current metadata object {}", + entry.metadata_location + ))); + }; + let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { + TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) + })?; + if !current_metadata.is_object() { + return Err(TableCatalogStoreError::Invalid(format!( + "current metadata {} must be a JSON object", + entry.metadata_location + ))); + } + + Ok(table_snapshot_expiration_report( + table_bucket, + &namespace, + &table, + &entry, + ¤t_metadata, + config, + OffsetDateTime::now_utc(), + )) + } + + pub(crate) async fn plan_table_compaction( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + config: TableCompactionPlanningConfig, + ) -> TableCatalogStoreResult { + validate_table_compaction_planning_config(&config)?; + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "current metadata location must be inside the table metadata directory".to_string(), + )); + } + + let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "current metadata object {}", + entry.metadata_location + ))); + }; + let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { + TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) + })?; + if !current_metadata.is_object() { + return Err(TableCatalogStoreError::Invalid(format!( + "current metadata {} must be a JSON object", + entry.metadata_location + ))); + } + + table_compaction_planning_report(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata, config).await + } + + pub(crate) async fn commit_table_compaction( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + config: TableCompactionPlanningConfig, + ) -> TableCatalogStoreResult { + validate_table_compaction_planning_config(&config)?; + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "current metadata location must be inside the table metadata directory".to_string(), + )); + } + + let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "current metadata object {}", + entry.metadata_location + ))); + }; + let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { + TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) + })?; + if !current_metadata.is_object() { + return Err(TableCatalogStoreError::Invalid(format!( + "current metadata {} must be a JSON object", + entry.metadata_location + ))); + } + let mut report = + table_compaction_planning_report(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata, config) + .await?; + if report.status != TableCompactionPlanningStatus::RewriteCandidates { + return Err(TableCatalogStoreError::Invalid("compaction has no safe rewrite candidates".to_string())); + } + let current_data_files = + compaction_current_data_files(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata).await?; + let current_data_files_by_key = current_data_files + .iter() + .map(|file| (file.object_key.as_str(), file)) + .collect::>(); + let rewritten_inputs = report + .rewrite_groups + .iter() + .flat_map(|group| group.input_file_locations.iter().cloned()) + .collect::>(); + let mut manifest_data_files = current_data_files + .iter() + .filter(|file| !rewritten_inputs.contains(&file.object_key)) + .cloned() + .collect::>(); + + let now = OffsetDateTime::now_utc(); + let snapshot_id = compaction_snapshot_id(¤t_metadata, &entry, now); + let sequence_number = next_compaction_sequence_number(¤t_metadata); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let warehouse_object_prefix = table_warehouse_object_prefix(&entry)?; + let compaction_id = Uuid::new_v4().to_string(); + let mut compacted_files = Vec::with_capacity(report.rewrite_groups.len()); + for rewrite_group in &mut report.rewrite_groups { + let output_prefix = rewrite_group + .input_file_locations + .first() + .and_then(|input| compaction_data_file_rewrite_prefix(&namespace, &table, Some(&warehouse_object_prefix), input)) + .ok_or_else(|| TableCatalogStoreError::Invalid("compaction rewrite group has no input files".to_string()))?; + let output_file = format!("{output_prefix}/compaction-{compaction_id}-{}.parquet", rewrite_group.group_id); + let output_file_path = table_object_s3_location(table_bucket, &output_file); + let (partition_spec_id, partition) = compaction_rewrite_group_partition(¤t_data_files_by_key, rewrite_group)?; + let sort_order_id = compaction_rewrite_group_sort_order(¤t_data_files_by_key, rewrite_group)?; + let mut input_files = Vec::with_capacity(rewrite_group.input_file_locations.len()); + for input_file in &rewrite_group.input_file_locations { + let Some(input_object) = self.backend.read_object(table_bucket, input_file).await? else { + return Err(TableCatalogStoreError::NotFound(format!("compaction input data file {input_file}"))); + }; + input_files.push((input_file.clone(), input_object.data)); + } + let compacted_file = compact_parquet_data_files(&input_files)?; + let output_bytes = u64::try_from(compacted_file.data.len()).unwrap_or(u64::MAX); + self.backend + .put_object(table_bucket, &output_file, compacted_file.data, TableCatalogPutPrecondition::IfAbsent) + .await?; + rewrite_group.output_file_location = Some(output_file_path.clone()); + rewrite_group.output_bytes = Some(output_bytes); + compacted_files.push(CompactedDataFile { + object_key: output_file, + file_path: output_file_path, + file_size_bytes: output_bytes, + record_count: compacted_file.record_count, + partition_spec_id, + partition, + sort_order_id, + status: 1, + snapshot_id, + sequence_number, + file_sequence_number: sequence_number, + }); + } + manifest_data_files.extend(compacted_files.iter().cloned()); + + let new_manifest = format!("{metadata_dir}/manifest-compaction-{compaction_id}.avro"); + let new_manifest_list = format!("{metadata_dir}/snap-{snapshot_id}-compaction-{compaction_id}.avro"); + let new_metadata = + default_table_metadata_file_path(&namespace, &table, &format!("compaction-{compaction_id}.metadata.json")); + let manifest_data = compacted_manifest_avro_bytes(&manifest_data_files)?; + let manifest_length = u64::try_from(manifest_data.len()).unwrap_or(u64::MAX); + self.backend + .put_object(table_bucket, &new_manifest, manifest_data, TableCatalogPutPrecondition::IfAbsent) + .await?; + let added_files_count = compacted_files.len(); + let added_rows_count = compacted_files + .iter() + .fold(0_u64, |rows, file| rows.saturating_add(file.record_count)); + let existing_files_count = manifest_data_files.len().saturating_sub(added_files_count); + let existing_rows_count = manifest_data_files + .iter() + .filter(|file| file.status == 0) + .fold(0_u64, |rows, file| rows.saturating_add(file.record_count)); + let manifest_list_data = compacted_manifest_list_avro_bytes(CompactionManifestListSummary { + manifest_path: &new_manifest, + manifest_length, + partition_spec_id: compaction_manifest_partition_spec_id(&manifest_data_files)?, + snapshot_id, + sequence_number, + added_files_count, + existing_files_count, + added_rows_count, + existing_rows_count, + })?; + self.backend + .put_object( + table_bucket, + &new_manifest_list, + manifest_list_data, + TableCatalogPutPrecondition::IfAbsent, + ) + .await?; + let new_metadata_data = compaction_metadata_json( + ¤t_metadata, + &entry, + snapshot_id, + sequence_number, + &new_manifest_list, + &entry.metadata_location, + now, + )?; + self.backend + .put_object(table_bucket, &new_metadata, new_metadata_data, TableCatalogPutPrecondition::IfAbsent) + .await?; + + let commit_result = self + .commit_table(TableCommitRequest { + table_bucket: table_bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: format!("compaction-{compaction_id}"), + idempotency_key: Some(format!("compaction-{compaction_id}")), + operation: "compaction".to_string(), + expected_version_token: entry.version_token, + expected_metadata_location: entry.metadata_location, + new_metadata_location: new_metadata.clone(), + requirements: Vec::new(), + writer: Some("rustfs-maintenance".to_string()), + }) + .await?; + + report.status = TableCompactionPlanningStatus::Committed; + report.committed_metadata_location = Some(commit_result.table.metadata_location); + for snapshot in &mut report.snapshot_reports { + if snapshot.status == TableCompactionPlanningStatus::RewriteCandidates { + snapshot.status = TableCompactionPlanningStatus::Committed; + if !snapshot.reasons.contains(&TableCompactionPlanningReason::CompactionCommitted) { + snapshot.reasons.push(TableCompactionPlanningReason::CompactionCommitted); + } + } + } + Ok(report) + } + + pub(crate) async fn export_table_catalog_entry( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + + let Some((table_bucket_entry, _)) = self + .read_entry::(self.catalog_bucket(), &self.paths.table_bucket_entry_path(table_bucket)) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + }; + let Some((namespace_entry, _)) = self + .read_entry::(self.catalog_bucket(), &self.paths.namespace_entry_path(table_bucket, &namespace)) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + table_bucket, + namespace.public_name() + ))); + }; + let Some((table_entry, _)) = self + .read_entry::(self.catalog_bucket(), &self.paths.table_entry_path(table_bucket, &namespace, &table)) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + let commit_recovery = self.table_commit_recovery_report_for_entry(&table_entry, 0).await?; + let backing_manifest = table_catalog_backing_manifest(&self.paths, &namespace, &table, &table_entry, &commit_recovery); + + Ok(TableCatalogExport { + table_bucket: table_bucket_entry, + namespace: namespace_entry, + table: table_entry, + backing_manifest, + }) + } + + pub(crate) async fn diagnose_table_catalog( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + retain_recent_metadata_files: usize, + ) -> TableCatalogStoreResult { + let parsed_namespace = parse_namespace_for_store(namespace)?; + let parsed_table = parse_table_for_store(table)?; + let catalog = self.export_table_catalog_entry(table_bucket, namespace, table).await?; + let current_metadata_location = catalog.table.metadata_location.clone(); + + let mut retained = BTreeSet::new(); + let mut current_metadata_for_refs = None; + let current_metadata_status = + if is_valid_table_metadata_location(&parsed_namespace, &parsed_table, ¤t_metadata_location) { + retained.insert(current_metadata_location.clone()); + match self.backend.read_object(table_bucket, ¤t_metadata_location).await? { + Some(current_metadata_object) => { + match serde_json::from_slice::(¤t_metadata_object.data) { + Ok(current_metadata) if current_metadata.is_object() => { + retained.extend(metadata_log_locations(¤t_metadata, &parsed_namespace, &parsed_table)); + current_metadata_for_refs = Some(current_metadata); + TableMetadataPointerStatus::Valid + } + Ok(_) | Err(_) => TableMetadataPointerStatus::InvalidJson, + } + } + None => TableMetadataPointerStatus::MissingObject, + } + } else { + TableMetadataPointerStatus::InvalidLocation + }; + + let mut metadata_locations = Vec::new(); + let metadata_prefix = format!("{}/", default_table_metadata_dir_path(&parsed_namespace, &parsed_table)); + for object in self.backend.list_objects(table_bucket, &metadata_prefix).await? { + if let Some(metadata_location) = metadata_location_from_metadata_file_path(&parsed_namespace, &parsed_table, &object) + { + metadata_locations.push(metadata_location); + } + } + metadata_locations.sort(); + metadata_locations.dedup(); + + for metadata_location in metadata_locations.iter().rev().take(retain_recent_metadata_files) { + retained.insert(metadata_location.clone()); + } + if let Some(current_metadata) = current_metadata_for_refs.as_ref() { + retained.extend( + metadata_locations_for_protected_snapshot_refs( + &self.backend, + table_bucket, + &parsed_namespace, + &parsed_table, + current_metadata, + &metadata_locations, + ) + .await?, + ); + } + + let orphan_metadata_candidate_locations = metadata_locations + .into_iter() + .filter(|metadata_location| !retained.contains(metadata_location)) + .collect(); + + let commit_recovery = self.plan_table_commit_recovery(table_bucket, namespace, table).await?; + let (recovery_status, recommended_actions) = table_catalog_recovery_summary(¤t_metadata_status, &commit_recovery); + let backing_manifest = + table_catalog_backing_manifest(&self.paths, &parsed_namespace, &parsed_table, &catalog.table, &commit_recovery); + + Ok(TableCatalogDiagnosticsReport { + catalog, + current_metadata_status, + recovery_status, + recommended_actions, + commit_recovery, + backing_manifest, + orphan_metadata_candidate_locations, + }) + } + + pub(crate) async fn plan_table_metadata_maintenance( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + retain_recent_metadata_files: usize, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "current metadata location must be inside the table metadata directory".to_string(), + )); + } + + let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "current metadata object {}", + entry.metadata_location + ))); + }; + let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { + TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) + })?; + if !current_metadata.is_object() { + return Err(TableCatalogStoreError::Invalid(format!( + "current metadata {} must be a JSON object", + entry.metadata_location + ))); + } + + let mut retained = BTreeSet::new(); + let mut maintenance_reasons = BTreeMap::>::new(); + for metadata_location in metadata_log_locations(¤t_metadata, &namespace, &table) { + retained.insert(metadata_location.clone()); + insert_metadata_maintenance_reason( + &mut maintenance_reasons, + metadata_location, + TableMetadataMaintenanceReason::MetadataLog, + ); + } + retained.insert(entry.metadata_location.clone()); + insert_metadata_maintenance_reason( + &mut maintenance_reasons, + entry.metadata_location.clone(), + TableMetadataMaintenanceReason::CurrentMetadata, + ); + + let mut metadata_locations = Vec::new(); + let metadata_prefix = format!("{}/", default_table_metadata_dir_path(&namespace, &table)); + for object in self.backend.list_objects(table_bucket, &metadata_prefix).await? { + if let Some(metadata_location) = metadata_location_from_metadata_file_path(&namespace, &table, &object) { + metadata_locations.push(metadata_location); + } + } + metadata_locations.sort(); + metadata_locations.dedup(); + let planned_metadata_file_count = metadata_locations.len(); + + for metadata_location in metadata_locations.iter().rev().take(retain_recent_metadata_files) { + retained.insert(metadata_location.clone()); + if metadata_location != &entry.metadata_location { + insert_metadata_maintenance_reason( + &mut maintenance_reasons, + metadata_location.clone(), + TableMetadataMaintenanceReason::RecentMetadata, + ); + } + } + for metadata_location in metadata_locations_for_protected_snapshot_refs( + &self.backend, + table_bucket, + &namespace, + &table, + ¤t_metadata, + &metadata_locations, + ) + .await? + { + retained.insert(metadata_location.clone()); + insert_metadata_maintenance_reason( + &mut maintenance_reasons, + metadata_location, + TableMetadataMaintenanceReason::ProtectedSnapshotRef, + ); + } + + let cleanup_candidate_locations = metadata_locations + .iter() + .filter(|metadata_location| !retained.contains(metadata_location.as_str())) + .cloned() + .collect::>(); + + let now = OffsetDateTime::now_utc(); + let mut deletable_metadata_locations = Vec::new(); + for metadata_location in &cleanup_candidate_locations { + insert_metadata_maintenance_reason( + &mut maintenance_reasons, + metadata_location.clone(), + TableMetadataMaintenanceReason::NoCurrentReachability, + ); + let Some(candidate_object) = self.backend.read_object(table_bucket, metadata_location).await? else { + insert_metadata_maintenance_reason( + &mut maintenance_reasons, + metadata_location.clone(), + TableMetadataMaintenanceReason::SafetyWindowPending, + ); + continue; + }; + if metadata_candidate_is_past_safety_window(candidate_object.mod_time, now) { + deletable_metadata_locations.push(metadata_location.clone()); + insert_metadata_maintenance_reason( + &mut maintenance_reasons, + metadata_location.clone(), + TableMetadataMaintenanceReason::SafetyWindowSatisfied, + ); + } else { + insert_metadata_maintenance_reason( + &mut maintenance_reasons, + metadata_location.clone(), + TableMetadataMaintenanceReason::SafetyWindowPending, + ); + } + } + let warehouse_object_prefix = table_warehouse_object_prefix(&entry).ok(); + let current_metadata_location = entry.metadata_location; + let retained_metadata_locations = retained.into_iter().collect::>(); + let object_reports = metadata_maintenance_object_reports(maintenance_reasons); + let referenced_object_reports = metadata_maintenance_referenced_object_reports( + &self.backend, + table_bucket, + &namespace, + &table, + warehouse_object_prefix.as_deref(), + ¤t_metadata, + &retained_metadata_locations, + ) + .await?; + let reachability_graph = + metadata_maintenance_reachability_graph_report(planned_metadata_file_count, &referenced_object_reports); + let (planned_object_file_count, cleanup_object_candidate_locations, deletable_object_locations, object_cleanup_reports) = + metadata_maintenance_object_cleanup_reports( + &self.backend, + table_bucket, + &namespace, + &table, + warehouse_object_prefix.as_deref(), + &referenced_object_reports, + now, + ) + .await?; + + let mut report = table_maintenance_report_with_recommended_actions(TableMetadataMaintenanceReport { + job: TableMetadataMaintenanceJob { + job_id: Uuid::new_v4().to_string(), + table_bucket: table_bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: entry.table_id, + operation: TableMetadataMaintenanceOperation::DryRun, + status: TableMetadataMaintenanceJobStatus::Successful, + failure_reason: None, + recommended_actions: Vec::new(), + config_source: TableMaintenanceConfigSource::Default, + scheduler_id: None, + scheduler_lease_id: String::new(), + scheduled_at: None, + worker_id: None, + lease_id: String::new(), + attempt: 0, + max_retry_attempts: 0, + next_retry_after: None, + quarantine_enabled: false, + quarantine_retention_seconds: 0, + heartbeat_at: None, + started_at: None, + finished_at: None, + current_metadata_location: current_metadata_location.clone(), + current_generation: entry.generation, + retain_recent_metadata_files, + safety_window_seconds: TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS, + cleanup_watermark_unix_seconds: (now - Duration::seconds(TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS)) + .unix_timestamp(), + planned_metadata_file_count, + retained_metadata_file_count: retained_metadata_locations.len(), + cleanup_candidate_count: cleanup_candidate_locations.len(), + deletable_metadata_file_count: deletable_metadata_locations.len(), + deleted_metadata_file_count: 0, + planned_object_file_count, + cleanup_candidate_object_count: cleanup_object_candidate_locations.len(), + deletable_object_count: deletable_object_locations.len(), + deleted_object_count: 0, + quarantined_object_count: 0, + }, + current_metadata_location, + retained_metadata_locations, + cleanup_candidate_locations, + deletable_metadata_locations, + cleanup_object_candidate_locations, + deletable_object_locations, + object_reports, + object_cleanup_reports, + referenced_object_reports, + reachability_graph, + snapshot_expiration: None, + compaction: None, + audit_events: Vec::new(), + }); + push_table_maintenance_audit_event( + &mut report, + now, + TableMaintenanceAuditActor::Scheduler, + TableMaintenanceAuditAction::Planned, + None, + None, + None, + ); + Ok(report) + } + + pub(crate) async fn delete_table_metadata_maintenance_candidates( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + retain_recent_metadata_files: usize, + ) -> TableCatalogStoreResult { + let report = self + .plan_table_metadata_maintenance(table_bucket, namespace, table, retain_recent_metadata_files) + .await?; + self.delete_table_metadata_maintenance_report(table_bucket, namespace, table, report) + .await + } + + pub(crate) async fn run_table_metadata_maintenance( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + delete: bool, + worker_id: Option, + ) -> TableCatalogStoreResult { + let effective = self + .get_effective_table_maintenance_config(table_bucket, namespace, table) + .await?; + self.run_table_metadata_maintenance_with_config(table_bucket, namespace, table, delete, worker_id, effective) + .await + } + + pub(crate) async fn run_table_metadata_maintenance_with_retention( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + delete: bool, + worker_id: Option, + retain_recent_metadata_files: usize, + ) -> TableCatalogStoreResult { + let mut effective = self + .get_effective_table_maintenance_config(table_bucket, namespace, table) + .await?; + effective.config.retain_recent_metadata_files = retain_recent_metadata_files; + self.run_table_metadata_maintenance_with_config(table_bucket, namespace, table, delete, worker_id, effective) + .await + } + + async fn run_table_metadata_maintenance_with_config( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + delete: bool, + worker_id: Option, + effective: TableMaintenanceEffectiveConfig, + ) -> TableCatalogStoreResult { + let mut report = self + .plan_table_metadata_maintenance(table_bucket, namespace, table, effective.config.retain_recent_metadata_files) + .await?; + + let started_at_time = OffsetDateTime::now_utc(); + let started_at = maintenance_timestamp(started_at_time); + report.job.operation = if delete { + TableMetadataMaintenanceOperation::Delete + } else { + TableMetadataMaintenanceOperation::DryRun + }; + report.job.status = TableMetadataMaintenanceJobStatus::Running; + report.job.failure_reason = None; + report.job.config_source = effective.source; + report.job.worker_id = worker_id; + report.job.lease_id = Uuid::new_v4().to_string(); + report.job.attempt = 1; + report.job.max_retry_attempts = effective.config.max_retry_attempts; + report.job.next_retry_after = None; + report.job.quarantine_enabled = effective.config.quarantine_enabled; + report.job.quarantine_retention_seconds = effective.config.quarantine_retention_seconds; + report.job.heartbeat_at = Some(started_at.clone()); + report.job.started_at = Some(started_at); + report.job.finished_at = None; + refresh_table_maintenance_report_recommended_actions(&mut report); + push_table_maintenance_audit_event( + &mut report, + started_at_time, + TableMaintenanceAuditActor::Worker, + TableMaintenanceAuditAction::WorkerStarted, + None, + Some(TableMetadataMaintenanceJobStatus::Successful), + Some(0), + ); + self.put_table_metadata_maintenance_report(&report).await?; + + self.finish_table_metadata_maintenance_run(table_bucket, namespace, table, delete, &effective, report) + .await + } + + async fn finish_table_metadata_maintenance_run( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + delete: bool, + effective: &TableMaintenanceEffectiveConfig, + mut report: TableMetadataMaintenanceReport, + ) -> TableCatalogStoreResult { + if delete && !effective.config.delete_enabled { + let finished_at = OffsetDateTime::now_utc(); + let before_status = Some(report.job.status.clone()); + let before_quarantined_object_count = Some(report.job.quarantined_object_count); + report.job.status = TableMetadataMaintenanceJobStatus::Failed; + report.job.failure_reason = Some(TABLE_MAINTENANCE_DELETE_DISABLED_REASON.to_string()); + apply_maintenance_retry_after(&mut report.job, &effective.config, finished_at); + report.job.finished_at = Some(maintenance_timestamp(finished_at)); + refresh_table_maintenance_report_recommended_actions(&mut report); + push_table_maintenance_audit_event( + &mut report, + finished_at, + TableMaintenanceAuditActor::Worker, + TableMaintenanceAuditAction::WorkerFailed, + Some(TABLE_MAINTENANCE_DELETE_DISABLED_REASON.to_string()), + before_status, + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&report).await?; + return Ok(report); + } + + if delete { + let running_report = report.clone(); + let mut deleted = match self + .delete_table_metadata_maintenance_report(table_bucket, namespace, table, report) + .await + { + Ok(report) => report, + Err(err) => { + let finished_at = OffsetDateTime::now_utc(); + let mut failed = running_report; + let before_status = Some(failed.job.status.clone()); + let before_quarantined_object_count = Some(failed.job.quarantined_object_count); + let reason = err.to_string(); + failed.job.status = TableMetadataMaintenanceJobStatus::Failed; + failed.job.failure_reason = Some(reason.clone()); + apply_maintenance_retry_after(&mut failed.job, &effective.config, finished_at); + failed.job.finished_at = Some(maintenance_timestamp(finished_at)); + refresh_table_maintenance_report_recommended_actions(&mut failed); + push_table_maintenance_audit_event( + &mut failed, + finished_at, + TableMaintenanceAuditActor::Worker, + TableMaintenanceAuditAction::WorkerFailed, + Some(reason), + before_status, + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&failed).await?; + return Err(err); + } + }; + let finished_at = OffsetDateTime::now_utc(); + let before_status = Some(TableMetadataMaintenanceJobStatus::Running); + let before_quarantined_object_count = Some(0); + deleted.job.finished_at = Some(maintenance_timestamp(finished_at)); + refresh_table_maintenance_report_recommended_actions(&mut deleted); + push_table_maintenance_audit_event( + &mut deleted, + finished_at, + TableMaintenanceAuditActor::Worker, + TableMaintenanceAuditAction::WorkerSucceeded, + None, + before_status, + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&deleted).await?; + return Ok(deleted); + } + + let finished_at = OffsetDateTime::now_utc(); + let before_status = Some(report.job.status.clone()); + let before_quarantined_object_count = Some(report.job.quarantined_object_count); + report.job.status = TableMetadataMaintenanceJobStatus::Successful; + report.job.finished_at = Some(maintenance_timestamp(finished_at)); + refresh_table_maintenance_report_recommended_actions(&mut report); + push_table_maintenance_audit_event( + &mut report, + finished_at, + TableMaintenanceAuditActor::Worker, + TableMaintenanceAuditAction::WorkerSucceeded, + None, + before_status, + before_quarantined_object_count, + ); + self.put_table_metadata_maintenance_report(&report).await?; + Ok(report) + } + + pub(in crate::table_catalog) async fn delete_table_metadata_maintenance_report( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + report: TableMetadataMaintenanceReport, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + if !is_valid_table_metadata_location(&namespace, &table, &report.current_metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "maintenance report current metadata location must be inside the table metadata directory".to_string(), + )); + } + + let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + if entry.metadata_location != report.current_metadata_location { + return Err(TableCatalogStoreError::Conflict( + "current metadata location changed before maintenance delete".to_string(), + )); + } + let warehouse_object_prefix = table_warehouse_object_prefix(&entry).ok(); + + let Some(current_metadata_object) = self.backend.read_object(table_bucket, &entry.metadata_location).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "current metadata object {}", + entry.metadata_location + ))); + }; + let current_metadata = serde_json::from_slice::(¤t_metadata_object.data).map_err(|err| { + TableCatalogStoreError::Invalid(format!("failed to parse current metadata {}: {err}", entry.metadata_location)) + })?; + if !current_metadata.is_object() { + return Err(TableCatalogStoreError::Invalid(format!( + "current metadata {} must be a JSON object", + entry.metadata_location + ))); + } + + let mut protected = metadata_log_locations(¤t_metadata, &namespace, &table); + protected.insert(entry.metadata_location.clone()); + protected.extend(report.retained_metadata_locations.iter().cloned()); + protected.extend( + metadata_locations_for_protected_snapshot_refs( + &self.backend, + table_bucket, + &namespace, + &table, + ¤t_metadata, + &report.cleanup_candidate_locations, + ) + .await?, + ); + + let cleanup_candidate_count = report.cleanup_candidate_locations.len(); + let planned_deletable_locations = report.deletable_metadata_locations.iter().cloned().collect::>(); + let mut cleanup_candidate_locations = BTreeSet::new(); + let now = OffsetDateTime::now_utc(); + for metadata_location in &report.cleanup_candidate_locations { + if !is_valid_table_metadata_location(&namespace, &table, metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "cleanup candidate {metadata_location} must be inside the table metadata directory" + ))); + } + if protected.contains(metadata_location.as_str()) { + return Err(TableCatalogStoreError::Conflict(format!( + "cleanup candidate {metadata_location} is retained by current metadata" + ))); + } + let Some(candidate_object) = self.backend.read_object(table_bucket, metadata_location).await? else { + continue; + }; + if !planned_deletable_locations.contains(metadata_location.as_str()) { + continue; + } + if !metadata_candidate_is_past_safety_window(candidate_object.mod_time, now) { + continue; + } + cleanup_candidate_locations.insert(metadata_location.clone()); + } + + let cleanup_candidate_locations = cleanup_candidate_locations.into_iter().collect::>(); + let deleted_locations = cleanup_candidate_locations.iter().cloned().collect::>(); + for metadata_location in &cleanup_candidate_locations { + self.backend.delete_object(table_bucket, metadata_location).await?; + } + + let referenced_object_reports = metadata_maintenance_referenced_object_reports( + &self.backend, + table_bucket, + &namespace, + &table, + warehouse_object_prefix.as_deref(), + ¤t_metadata, + &report.retained_metadata_locations, + ) + .await?; + let referenced_object_locations = if referenced_object_reports + .iter() + .any(|report| report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired) + { + BTreeSet::new() + } else { + referenced_object_reports + .iter() + .filter_map(|report| table_catalog_object_key_from_location(table_bucket, &report.object_location)) + .collect::>() + }; + let planned_deletable_object_locations = report.deletable_object_locations.iter().cloned().collect::>(); + let mut cleanup_object_candidate_locations = BTreeSet::new(); + if !referenced_object_reports + .iter() + .any(|report| report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired) + { + for object_location in &report.cleanup_object_candidate_locations { + if table_maintenance_object_kind(&namespace, &table, warehouse_object_prefix.as_deref(), object_location) + .is_none() + { + return Err(TableCatalogStoreError::Invalid(format!( + "cleanup object candidate {object_location} must be inside table metadata, data, or delete directories" + ))); + } + if referenced_object_locations.contains(object_location.as_str()) { + return Err(TableCatalogStoreError::Conflict(format!( + "cleanup object candidate {object_location} is retained by current metadata" + ))); + } + let Some(candidate_object) = self.backend.read_object(table_bucket, object_location).await? else { + continue; + }; + if !planned_deletable_object_locations.contains(object_location.as_str()) { + continue; + } + if !metadata_candidate_is_past_safety_window(candidate_object.mod_time, now) { + continue; + } + cleanup_object_candidate_locations.insert(object_location.clone()); + } + } + + let cleanup_object_candidate_locations = cleanup_object_candidate_locations.into_iter().collect::>(); + let deleted_object_locations = cleanup_object_candidate_locations.iter().cloned().collect::>(); + for object_location in &cleanup_object_candidate_locations { + self.backend.delete_object(table_bucket, object_location).await?; + } + + let retained_metadata_locations = protected.into_iter().collect::>(); + let mut job = report.job; + job.operation = TableMetadataMaintenanceOperation::Delete; + job.status = TableMetadataMaintenanceJobStatus::Successful; + job.failure_reason = None; + job.retained_metadata_file_count = retained_metadata_locations.len(); + job.cleanup_candidate_count = cleanup_candidate_count; + job.deletable_metadata_file_count = planned_deletable_locations.len(); + job.deleted_metadata_file_count = cleanup_candidate_locations.len(); + job.cleanup_candidate_object_count = report.cleanup_object_candidate_locations.len(); + job.deletable_object_count = planned_deletable_object_locations.len(); + job.deleted_object_count = cleanup_object_candidate_locations.len(); + let mut object_reports = report.object_reports; + mark_deleted_metadata_object_reports(&mut object_reports, &deleted_locations); + let mut object_cleanup_reports = report.object_cleanup_reports; + mark_deleted_object_cleanup_reports(&mut object_cleanup_reports, &deleted_object_locations); + + Ok(table_maintenance_report_with_recommended_actions(TableMetadataMaintenanceReport { + job, + current_metadata_location: entry.metadata_location, + retained_metadata_locations, + cleanup_candidate_locations: cleanup_candidate_locations.clone(), + deletable_metadata_locations: cleanup_candidate_locations, + cleanup_object_candidate_locations: cleanup_object_candidate_locations.clone(), + deletable_object_locations: cleanup_object_candidate_locations, + object_reports, + object_cleanup_reports, + referenced_object_reports, + reachability_graph: report.reachability_graph, + snapshot_expiration: report.snapshot_expiration, + compaction: report.compaction, + audit_events: report.audit_events, + })) + } +} + +#[async_trait::async_trait] +impl TableCatalogStore for ObjectTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult> { + self.read_entry::(self.catalog_bucket(), &self.paths.table_bucket_entry_path(table_bucket)) + .await + .map(|entry| entry.map(|(bucket, _)| bucket)) + } + + async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("table bucket", entry.version)?; + if entry.table_bucket.is_empty() { + return Err(TableCatalogStoreError::Invalid("table bucket name cannot be empty".to_string())); + } + if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE { + return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string())); + } + let _registry_guard = self.acquire_table_bucket_registry_write_permit().await?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; + let object = self.paths.table_bucket_entry_path(&entry.table_bucket); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; + self.write_entry_unlocked(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::Any) + .await + } + + async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("namespace", entry.version)?; + self.require_table_bucket(&entry.table_bucket).await?; + let namespace = parse_namespace_for_store(&entry.namespace)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; + let bucket_path = self.paths.table_bucket_entry_path(&entry.table_bucket); + let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?; + let object = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); + self.write_entry(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::IfAbsent) + .await + } + + async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult> { + let mut entries = Vec::new(); + for object in self + .backend + .list_objects(self.catalog_bucket(), &self.paths.namespace_entries_prefix(table_bucket)) + .await? + { + if !object.ends_with(NAMESPACE_ENTRY_FILE) { + continue; + } + if let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? { + entries.push(entry); + } + } + entries.sort_by(|left, right| left.namespace.cmp(&right.namespace)); + Ok(entries) + } + + async fn list_namespaces_page( + &self, + table_bucket: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + self.list_entry_page(&self.paths.namespace_entries_prefix(table_bucket), NAMESPACE_ENTRY_FILE, cursor, limit) + .await + } + + async fn get_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + self.read_entry::(self.catalog_bucket(), &self.paths.namespace_entry_path(table_bucket, &namespace)) + .await + .map(|entry| entry.map(|(namespace, _)| namespace)) + } + + async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()> { + let namespace = parse_namespace_for_store(namespace)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + let bucket_path = self.paths.table_bucket_entry_path(table_bucket); + let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?; + let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); + // Match create_namespace and migration lock order while draining table/view creation. + let _namespace_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &namespace_path) + .await?; + if self + .read_entry_unlocked::(self.catalog_bucket(), &namespace_path) + .await? + .is_none() + { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + table_bucket, + namespace.public_name() + ))); + } + if !self.list_tables(table_bucket, &namespace.public_name()).await?.is_empty() { + return Err(TableCatalogStoreError::Conflict(format!( + "namespace {}/{} is not empty", + table_bucket, + namespace.public_name() + ))); + } + if !self.list_views(table_bucket, &namespace.public_name()).await?.is_empty() { + return Err(TableCatalogStoreError::Conflict(format!( + "namespace {}/{} is not empty", + table_bucket, + namespace.public_name() + ))); + } + self.backend + .delete_object_unlocked(self.catalog_bucket(), &namespace_path) + .await + } + + async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { + self.write_table_entry(entry, TableCatalogPutPrecondition::IfAbsent).await + } + + async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { + self.write_table_entry(entry, TableCatalogPutPrecondition::IfAbsent).await + } + + async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + let mut entries = Vec::new(); + for object in self + .backend + .list_objects(self.catalog_bucket(), &self.paths.table_entries_prefix(table_bucket, &namespace)) + .await? + { + if !object.ends_with(TABLE_ENTRY_FILE) { + continue; + } + if let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? { + entries.push(entry); + } + } + entries.sort_by(|left, right| left.table.cmp(&right.table)); + Ok(entries) + } + + async fn list_tables_page( + &self, + table_bucket: &str, + namespace: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + self.list_entry_page( + &self.paths.table_entries_prefix(table_bucket, &namespace), + TABLE_ENTRY_FILE, + cursor, + limit, + ) + .await + } + + async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + self.read_entry::(self.catalog_bucket(), &self.paths.table_entry_path(table_bucket, &namespace, &table)) + .await + .map(|entry| entry.map(|(table, _)| table)) + } + + async fn resolve_table_data_plane_resource( + &self, + table_bucket: &str, + object: &str, + ) -> TableCatalogStoreResult> { + if table_bucket.is_empty() || object.is_empty() { + return Ok(None); + } + let Some(table_bucket_entry) = self.get_table_bucket(table_bucket).await? else { + return Ok(None); + }; + if table_bucket_entry.state != TableCatalogEntryState::Active { + return Ok(None); + } + + if self.warehouse_index_ready(table_bucket).await? { + return self.resolve_table_data_plane_resource_from_index(table_bucket, object).await; + } + + match self.backfill_table_warehouse_index(table_bucket).await { + Ok(()) => self.resolve_table_data_plane_resource_from_index(table_bucket, object).await, + Err(err) => { + tracing::warn!( + table_bucket = %table_bucket, + error = %err, + "failed to backfill table warehouse index; falling back to catalog scan" + ); + scan_table_data_plane_resource_for_object(self, table_bucket, object).await + } + } + } + + async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { + let commit_started = Instant::now(); + record_table_commit_attempt(&request.operation); + let namespace = parse_namespace_for_store(&request.namespace)?; + let table = parse_table_for_store(&request.table)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; + let table_path = self.paths.table_entry_path(&request.table_bucket, &namespace, &table); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + + let Some((current, current_etag)) = self + .read_table_with_etag_unlocked(&request.table_bucket, &namespace, &table) + .await? + else { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + request.table_bucket, request.namespace, request.table + ))), + ); + }; + + let commit_path = self + .paths + .commit_log_entry_path(&request.table_bucket, ¤t.table_id, &request.commit_id); + let existing_commit = self.read_commit_by_path(&commit_path).await?; + let idempotency_path = request.idempotency_key.as_deref().map(|idempotency_key| { + self.paths + .commit_idempotency_entry_path(&request.table_bucket, ¤t.table_id, idempotency_key) + }); + let existing_idempotency_commit = match idempotency_path.as_deref() { + Some(idempotency_path) => self.read_commit_by_path(idempotency_path).await?, + None => None, + }; + + if let Some(existing) = existing_commit.as_ref() { + if !commit_log_matches_request(existing, &request, ¤t.table_id) { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict(format!( + "commit id already exists: {}", + request.commit_id + ))), + ); + } + if matches!(existing.status, CommitLogStatus::Committed) || table_matches_committed_log(¤t, existing) { + let mut committed = existing.clone(); + committed.status = CommitLogStatus::Committed; + let _ = self + .finalize_commit_log(&commit_path, idempotency_path.as_deref(), &committed) + .await; + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Ok(TableCommitResult { + table: current, + commit_log: committed, + }), + ); + } + if !matches!(existing.status, CommitLogStatus::Staged) || !table_matches_staged_base(¤t, existing) { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict( + "existing commit record does not match current table state".to_string(), + )), + ); + } + } + if let Some(existing) = existing_idempotency_commit.as_ref() + && !commit_log_matches_request(existing, &request, ¤t.table_id) + { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict("idempotency key already exists".to_string())), + ); + } + if existing_commit.is_none() && existing_idempotency_commit.is_some() { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict( + "idempotency key exists without a recoverable commit record".to_string(), + )), + ); + } + + if current.version_token != request.expected_version_token { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict( + "current table version token does not match expected token".to_string(), + )), + ); + } + if current.metadata_location != request.expected_metadata_location { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict( + "current table metadata location does not match expected location".to_string(), + )), + ); + } + if !is_valid_table_metadata_location(&namespace, &table, &request.new_metadata_location) { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Invalid( + "new metadata location must be inside the table metadata directory".to_string(), + )), + ); + } + let Some(new_metadata_object) = self + .backend + .read_object(&request.table_bucket, &request.new_metadata_location) + .await? + else { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::NotFound(format!( + "new metadata object {}", + request.new_metadata_location + ))), + ); + }; + let next_warehouse_location = + table_metadata_warehouse_location(&request.table_bucket, &request.new_metadata_location, &new_metadata_object)?; + + let has_existing_commit = existing_commit.is_some(); + let mut staged_commit_log = existing_commit.unwrap_or_else(|| CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: request.commit_id.clone(), + idempotency_key: request.idempotency_key.clone(), + table_id: current.table_id.clone(), + operation: request.operation.clone(), + expected_version_token: request.expected_version_token.clone(), + new_version_token: format!("token-{}", Uuid::new_v4()), + previous_metadata_location: current.metadata_location.clone(), + new_metadata_location: request.new_metadata_location.clone(), + requirements: request.requirements.clone(), + status: CommitLogStatus::Staged, + writer: request.writer.clone(), + created_at: None, + updated_at: None, + }); + staged_commit_log.status = CommitLogStatus::Staged; + + let mut next = current.clone(); + next.metadata_location = staged_commit_log.new_metadata_location.clone(); + if let Some(warehouse_location) = next_warehouse_location { + next.warehouse_location = warehouse_location; + } + next.version_token = staged_commit_log.new_version_token.clone(); + next.generation = current.generation.saturating_add(1); + let reservation = self.reserve_table_warehouse_index(&next).await?; + + let staged_write_result = async { + if !has_existing_commit { + self.write_entry( + self.catalog_bucket(), + &commit_path, + &staged_commit_log, + TableCatalogPutPrecondition::IfAbsent, + ) + .await?; + } + if let Some(idempotency_path) = idempotency_path.as_deref() + && existing_idempotency_commit.is_none() + { + self.write_entry( + self.catalog_bucket(), + idempotency_path, + &staged_commit_log, + TableCatalogPutPrecondition::IfAbsent, + ) + .await?; + } + Ok(()) + } + .await; + if let Err(err) = staged_write_result { + self.delete_created_table_warehouse_index(&next, reservation, "commit staging failed") + .await; + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(err), + ); + } + + let cas_started = Instant::now(); + let cas_result = self + .write_entry_unlocked( + self.catalog_bucket(), + &table_path, + &next, + TableCatalogPutPrecondition::IfMatch(current_etag), + ) + .await; + record_table_commit_cas_result(&request.operation, cas_started, &cas_result); + if let Err(err) = cas_result { + self.delete_created_table_warehouse_index(&next, reservation, "table pointer CAS failed") + .await; + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(err), + ); + } + self.delete_table_warehouse_index_if_changed(¤t, &next).await; + + let mut commit_log = staged_commit_log; + commit_log.status = CommitLogStatus::Committed; + // After the table CAS succeeds, the staged record is the durable recovery source. + // A finalization failure must not turn an externally committed pointer into a failed commit response. + let _ = self + .finalize_commit_log(&commit_path, idempotency_path.as_deref(), &commit_log) + .await; + + table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Ok(TableCommitResult { table: next, commit_log }), + ) + } + + async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); + let _namespace_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &namespace_path) + .await?; + let object = self.paths.table_entry_path(table_bucket, &namespace, &table); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; + let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await?; + if let Err(err) = self.delete_table_warehouse_index(&entry).await { + if let Err(restore_err) = self + .write_entry_unlocked(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::IfAbsent) + .await + { + tracing::warn!( + table_bucket = %entry.table_bucket, + namespace = %entry.namespace, + table = %entry.table, + table_id = %entry.table_id, + error = %restore_err, + "failed to restore table entry after warehouse index delete failure" + ); + } + tracing::warn!( + table_bucket = %entry.table_bucket, + namespace = %entry.namespace, + table = %entry.table, + table_id = %entry.table_id, + error = %err, + "failed to delete table warehouse index after dropping table" + ); + return Err(err); + } + Ok(()) + } + + async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { + self.write_view_entry(entry, TableCatalogPutPrecondition::IfAbsent).await + } + + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + let mut entries = Vec::new(); + for object in self + .backend + .list_objects(self.catalog_bucket(), &self.paths.view_entries_prefix(table_bucket, &namespace)) + .await? + { + if !object.ends_with(VIEW_ENTRY_FILE) { + continue; + } + if let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? { + entries.push(entry); + } + } + entries.sort_by(|left, right| left.view.cmp(&right.view)); + Ok(entries) + } + + async fn list_views_page( + &self, + table_bucket: &str, + namespace: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + self.list_entry_page(&self.paths.view_entries_prefix(table_bucket, &namespace), VIEW_ENTRY_FILE, cursor, limit) + .await + } + + async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + let view = parse_table_for_store(view)?; + self.read_entry::(self.catalog_bucket(), &self.paths.view_entry_path(table_bucket, &namespace, &view)) + .await + .map(|entry| entry.map(|(view, _)| view)) + } + + async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(&request.namespace)?; + let view = parse_table_for_store(&request.view)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; + let namespace_path = self.paths.namespace_entry_path(&request.table_bucket, &namespace); + let _namespace_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &namespace_path) + .await?; + let view_path = self.paths.view_entry_path(&request.table_bucket, &namespace, &view); + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &view_path).await?; + let Some((current, current_etag)) = self + .read_view_with_etag_unlocked(&request.table_bucket, &namespace, &view) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + request.table_bucket, request.namespace, request.view + ))); + }; + if current.version_token != request.expected_version_token { + return Err(TableCatalogStoreError::Conflict( + "current view version token does not match expected token".to_string(), + )); + } + if current.metadata_location != request.expected_metadata_location { + return Err(TableCatalogStoreError::Conflict( + "current view metadata location does not match expected location".to_string(), + )); + } + if !is_valid_view_metadata_location(&namespace, &view, &request.new_metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "new metadata location must be inside the view metadata directory".to_string(), + )); + } + let Some(new_metadata_object) = self + .backend + .read_object(&request.table_bucket, &request.new_metadata_location) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "new view metadata object {}", + request.new_metadata_location + ))); + }; + let next_warehouse_location = + view_metadata_warehouse_location(&request.table_bucket, &request.new_metadata_location, &new_metadata_object)?; + + let mut next = current; + next.metadata_location = request.new_metadata_location; + if let Some(warehouse_location) = next_warehouse_location { + next.warehouse_location = warehouse_location; + } + next.version_token = format!("token-{}", Uuid::new_v4()); + next.generation = next.generation.saturating_add(1); + self.write_entry_unlocked( + self.catalog_bucket(), + &view_path, + &next, + TableCatalogPutPrecondition::IfMatch(current_etag), + ) + .await?; + Ok(ViewCommitResult { view: next }) + } + + async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { + let namespace = parse_namespace_for_store(namespace)?; + let view = parse_table_for_store(view)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); + let _namespace_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &namespace_path) + .await?; + let object = self.paths.view_entry_path(table_bucket, &namespace, &view); + let _view_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; + if self + .read_entry_unlocked::(self.catalog_bucket(), &object) + .await? + .is_none() + { + return Err(TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + table_bucket, + namespace.public_name(), + view.as_str() + ))); + } + self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await + } + + async fn get_commit_by_id( + &self, + table_bucket: &str, + table_id: &str, + commit_id: &str, + ) -> TableCatalogStoreResult> { + let object = self.paths.commit_log_entry_path(table_bucket, table_id, commit_id); + self.read_commit_by_path(&object).await + } + + async fn get_commit_by_idempotency_key( + &self, + table_bucket: &str, + table_id: &str, + idempotency_key: &str, + ) -> TableCatalogStoreResult> { + let object = self + .paths + .commit_idempotency_entry_path(table_bucket, table_id, idempotency_key); + self.read_commit_by_path(&object).await + } +} diff --git a/rustfs/src/table_catalog/store/strong.rs b/rustfs/src/table_catalog/store/strong.rs new file mode 100644 index 000000000..acc0627ac --- /dev/null +++ b/rustfs/src/table_catalog/store/strong.rs @@ -0,0 +1,1363 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +pub(in crate::table_catalog) type StrongNamespaceKey = (String, String); +type StrongResourceKey = (String, String, String); +type StrongCommitKey = (String, String, String); +type StrongWarehouseIndex = BTreeMap>; + +#[derive(Clone, Default)] +pub(in crate::table_catalog) struct StrongTableCatalogState { + pub(super) hydrated: bool, + pub(super) snapshot_etag: Option, + pub(super) table_buckets: BTreeMap, + pub(in crate::table_catalog) namespaces: BTreeMap, + pub(super) tables: BTreeMap, + pub(super) views: BTreeMap, + pub(super) commits: BTreeMap, + pub(super) idempotency: BTreeMap, + pub(super) warehouse_index: StrongWarehouseIndex, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(in crate::table_catalog) struct StrongCommitSnapshotRecord { + pub(super) table_bucket: String, + pub(super) table_id: String, + pub(super) lookup_key: String, + pub(super) commit: CommitLogEntry, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(in crate::table_catalog) struct StrongTableCatalogSnapshot { + pub(in crate::table_catalog) version: u16, + pub(in crate::table_catalog) table_buckets: Vec, + pub(in crate::table_catalog) namespaces: Vec, + pub(in crate::table_catalog) tables: Vec, + pub(in crate::table_catalog) views: Vec, + pub(in crate::table_catalog) commits: Vec, + pub(in crate::table_catalog) idempotency: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub(super) struct StrongTableCatalogBucketSnapshot { + pub(super) table_bucket: TableBucketEntry, + pub(super) namespaces: Vec, + pub(super) tables: Vec, + pub(super) views: Vec, + pub(super) commits: Vec, + pub(super) idempotency: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(super) enum TableCatalogBackingMigrationFenceStatus { + Preparing, + Materialized, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TableCatalogBackingMigrationGlobalFence { + pub(super) version: u16, + pub(super) migration_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TableCatalogBackingMigrationFence { + pub(super) version: u16, + pub(super) table_bucket: String, + pub(super) migration_id: String, + pub(super) status: TableCatalogBackingMigrationFenceStatus, + pub(super) target_bucket_existed: bool, + pub(super) source_fingerprint: Option, + pub(super) target_snapshot_etag: Option, +} + +pub(super) fn table_catalog_bucket_snapshot_fingerprint( + snapshot: &StrongTableCatalogBucketSnapshot, +) -> TableCatalogStoreResult { + let data = serde_json::to_vec(snapshot) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode catalog migration snapshot: {err}")))?; + Ok(hex_simd::encode_to_string(Sha256::digest(data), hex_simd::AsciiCase::Lower)) +} + +#[derive(Clone)] +pub(crate) struct StrongTableCatalogStore { + object_backend: B, + // Single mutex protecting all catalog state (table_buckets, namespaces, tables, views, commits, idempotency). + // This is intentional: many operations require atomic read-modify-write across multiple fields. + // Splitting into per-field locks would introduce deadlock risk and complexity. + // If lock contention becomes a bottleneck (acquisition time > 10ms), consider: + // 1. Using RwLock for read-heavy paths (but most paths need write access) + // 2. Splitting into logical groups (e.g., metadata vs commits) + // 3. Using optimistic concurrency with version checks + pub(in crate::table_catalog) state: Arc>, + // Serializes local snapshot mutations; object ETags fence independent store instances. + write_lock: Arc>, +} + +impl StrongTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + pub fn new(object_backend: B) -> Self { + Self { + object_backend, + state: Arc::new(tokio::sync::Mutex::new(StrongTableCatalogState::default())), + write_lock: Arc::new(tokio::sync::Mutex::new(())), + } + } + + pub(in crate::table_catalog) fn namespace_key(table_bucket: &str, namespace: &Namespace) -> StrongNamespaceKey { + (table_bucket.to_string(), namespace.public_name()) + } + + fn table_key(table_bucket: &str, namespace: &Namespace, table: &IdentifierSegment) -> StrongResourceKey { + (table_bucket.to_string(), namespace.public_name(), table.as_str().to_string()) + } + + fn commit_key(table_bucket: &str, table_id: &str, commit_id: &str) -> StrongCommitKey { + (table_bucket.to_string(), table_id.to_string(), commit_id.to_string()) + } + + fn idempotency_key(table_bucket: &str, table_id: &str, idempotency_key: &str) -> StrongCommitKey { + (table_bucket.to_string(), table_id.to_string(), idempotency_key.to_string()) + } + + pub(in crate::table_catalog) fn snapshot_object_path() -> String { + format!("{INTERNAL_CATALOG_ROOT}/{STRONG_TABLE_CATALOG_BACKING_ROOT}/{STRONG_TABLE_CATALOG_SNAPSHOT_FILE}") + } + + fn snapshot_from_state_locked(state: &StrongTableCatalogState) -> StrongTableCatalogSnapshot { + StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_VERSION, + table_buckets: state.table_buckets.values().cloned().collect(), + namespaces: state.namespaces.values().cloned().collect(), + tables: state.tables.values().cloned().collect(), + views: state.views.values().cloned().collect(), + commits: state + .commits + .iter() + .map(|((table_bucket, table_id, lookup_key), commit)| StrongCommitSnapshotRecord { + table_bucket: table_bucket.clone(), + table_id: table_id.clone(), + lookup_key: lookup_key.clone(), + commit: commit.clone(), + }) + .collect(), + idempotency: state + .idempotency + .iter() + .map(|((table_bucket, table_id, lookup_key), commit)| StrongCommitSnapshotRecord { + table_bucket: table_bucket.clone(), + table_id: table_id.clone(), + lookup_key: lookup_key.clone(), + commit: commit.clone(), + }) + .collect(), + } + } + + fn bucket_snapshot_from_state_locked( + state: &StrongTableCatalogState, + table_bucket: &str, + ) -> Option { + let table_bucket_entry = state.table_buckets.get(table_bucket)?.clone(); + Some(StrongTableCatalogBucketSnapshot { + table_bucket: table_bucket_entry, + namespaces: state + .namespaces + .iter() + .filter(|((entry_bucket, _), _)| entry_bucket == table_bucket) + .map(|(_, entry)| entry.clone()) + .collect(), + tables: state + .tables + .iter() + .filter(|((entry_bucket, _, _), _)| entry_bucket == table_bucket) + .map(|(_, entry)| entry.clone()) + .collect(), + views: state + .views + .iter() + .filter(|((entry_bucket, _, _), _)| entry_bucket == table_bucket) + .map(|(_, entry)| entry.clone()) + .collect(), + commits: state + .commits + .iter() + .filter(|((entry_bucket, _, _), _)| entry_bucket == table_bucket) + .map(|((entry_bucket, table_id, lookup_key), commit)| StrongCommitSnapshotRecord { + table_bucket: entry_bucket.clone(), + table_id: table_id.clone(), + lookup_key: lookup_key.clone(), + commit: commit.clone(), + }) + .collect(), + idempotency: state + .idempotency + .iter() + .filter(|((entry_bucket, _, _), _)| entry_bucket == table_bucket) + .map(|((entry_bucket, table_id, lookup_key), commit)| StrongCommitSnapshotRecord { + table_bucket: entry_bucket.clone(), + table_id: table_id.clone(), + lookup_key: lookup_key.clone(), + commit: commit.clone(), + }) + .collect(), + }) + } + + fn remove_bucket_from_state_locked(state: &mut StrongTableCatalogState, table_bucket: &str) { + state.table_buckets.remove(table_bucket); + state.namespaces.retain(|(entry_bucket, _), _| entry_bucket != table_bucket); + state.tables.retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); + state.views.retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); + state.commits.retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); + state + .idempotency + .retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); + state.warehouse_index.remove(table_bucket); + } + + pub(super) fn insert_bucket_snapshot_locked( + state: &mut StrongTableCatalogState, + snapshot: StrongTableCatalogBucketSnapshot, + ) -> TableCatalogStoreResult<()> { + let table_bucket = snapshot.table_bucket.table_bucket.clone(); + Self::remove_bucket_from_state_locked(state, &table_bucket); + state.table_buckets.insert(table_bucket.clone(), snapshot.table_bucket); + for entry in snapshot.namespaces { + let namespace = parse_namespace_for_store(&entry.namespace)?; + state.namespaces.insert(Self::namespace_key(&table_bucket, &namespace), entry); + } + for entry in snapshot.tables { + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + state.tables.insert(Self::table_key(&table_bucket, &namespace, &table), entry); + } + for entry in snapshot.views { + let namespace = parse_namespace_for_store(&entry.namespace)?; + let view = parse_table_for_store(&entry.view)?; + state.views.insert(Self::table_key(&table_bucket, &namespace, &view), entry); + } + for record in snapshot.commits { + state.commits.insert( + Self::commit_key(&record.table_bucket, &record.table_id, &record.lookup_key), + record.commit, + ); + } + for record in snapshot.idempotency { + state.idempotency.insert( + Self::idempotency_key(&record.table_bucket, &record.table_id, &record.lookup_key), + record.commit, + ); + } + Self::rebuild_warehouse_index_locked(state) + } + + fn rebuild_warehouse_index_locked(state: &mut StrongTableCatalogState) -> TableCatalogStoreResult<()> { + let mut warehouse_index: StrongWarehouseIndex = BTreeMap::new(); + for ((table_bucket, namespace, table), entry) in &state.tables { + if entry.state != TableCatalogEntryState::Active { + continue; + } + if !state + .table_buckets + .get(table_bucket) + .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) + { + continue; + } + if !state + .namespaces + .get(&(table_bucket.clone(), namespace.clone())) + .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) + { + continue; + } + let Ok(warehouse_object_prefix) = table_warehouse_object_prefix(entry) else { + continue; + }; + let table_key = (table_bucket.clone(), namespace.clone(), table.clone()); + if let Some(existing_key) = warehouse_index + .entry(table_bucket.clone()) + .or_default() + .insert(warehouse_object_prefix.clone(), table_key.clone()) + { + return Err(TableCatalogStoreError::Invalid(format!( + "duplicate active table warehouse location in strong catalog snapshot: {warehouse_object_prefix} is owned by {}/{}/{} and {}/{}/{}", + existing_key.0, existing_key.1, existing_key.2, table_key.0, table_key.1, table_key.2 + ))); + } + } + state.warehouse_index = warehouse_index; + Ok(()) + } + + fn snapshot_from_mutated_state_locked( + state: &mut StrongTableCatalogState, + ) -> TableCatalogStoreResult { + Self::rebuild_warehouse_index_locked(state)?; + Ok(Self::snapshot_from_state_locked(state)) + } + + fn state_from_snapshot( + snapshot: StrongTableCatalogSnapshot, + snapshot_etag: Option, + ) -> TableCatalogStoreResult { + if snapshot.version != STRONG_TABLE_CATALOG_SNAPSHOT_VERSION { + return Err(TableCatalogStoreError::Invalid(format!( + "unsupported strong catalog snapshot version: {}", + snapshot.version + ))); + } + + let mut state = StrongTableCatalogState { + hydrated: true, + snapshot_etag, + ..StrongTableCatalogState::default() + }; + for entry in snapshot.table_buckets { + state.table_buckets.insert(entry.table_bucket.clone(), entry); + } + for entry in snapshot.namespaces { + let namespace = parse_namespace_for_store(&entry.namespace)?; + state + .namespaces + .insert(Self::namespace_key(&entry.table_bucket, &namespace), entry); + } + for entry in snapshot.tables { + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + state + .tables + .insert(Self::table_key(&entry.table_bucket, &namespace, &table), entry); + } + for entry in snapshot.views { + let namespace = parse_namespace_for_store(&entry.namespace)?; + let view = parse_table_for_store(&entry.view)?; + state + .views + .insert(Self::table_key(&entry.table_bucket, &namespace, &view), entry); + } + for record in snapshot.commits { + state.commits.insert( + Self::commit_key(&record.table_bucket, &record.table_id, &record.lookup_key), + record.commit, + ); + } + for record in snapshot.idempotency { + state.idempotency.insert( + Self::idempotency_key(&record.table_bucket, &record.table_id, &record.lookup_key), + record.commit, + ); + } + Self::rebuild_warehouse_index_locked(&mut state)?; + Ok(state) + } + + fn snapshot_write_precondition_locked(state: &StrongTableCatalogState) -> TableCatalogPutPrecondition { + state + .snapshot_etag + .as_ref() + .map_or(TableCatalogPutPrecondition::IfAbsent, |etag| { + TableCatalogPutPrecondition::IfMatch(etag.clone()) + }) + } + + fn snapshot_draft_context_locked(state: &StrongTableCatalogState) -> (TableCatalogPutPrecondition, StrongTableCatalogState) { + (Self::snapshot_write_precondition_locked(state), state.clone()) + } + + async fn hydrate_state(&self) -> TableCatalogStoreResult<()> { + let Some(current_snapshot_etag) = ({ + let state = self.state.lock().await; + if state.hydrated { + Some(state.snapshot_etag.clone()) + } else { + None + } + }) else { + return self.reload_state_from_durable().await; + }; + + let snapshot_metadata = self + .object_backend + .object_metadata(RUSTFS_META_BUCKET, &Self::snapshot_object_path()) + .await?; + match (snapshot_metadata, current_snapshot_etag.as_deref()) { + (None, None) => Ok(()), + (Some(metadata), Some(current_etag)) if metadata.etag.as_deref() == Some(current_etag) => Ok(()), + _ => self.reload_state_from_durable().await, + } + } + + async fn reload_state_from_durable(&self) -> TableCatalogStoreResult<()> { + let snapshot_object = self + .object_backend + .read_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path()) + .await?; + let mut state = self.state.lock().await; + if let Some(snapshot_object) = snapshot_object { + let snapshot = serde_json::from_slice::(&snapshot_object.data) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to decode strong catalog snapshot: {err}")))?; + *state = Self::state_from_snapshot(snapshot, snapshot_object.etag)?; + } else { + *state = StrongTableCatalogState { + hydrated: true, + ..StrongTableCatalogState::default() + }; + } + Ok(()) + } + + async fn persist_snapshot( + &self, + snapshot: StrongTableCatalogSnapshot, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> { + let data = serde_json::to_vec(&snapshot) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode strong catalog snapshot: {err}")))?; + self.object_backend + .put_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), data, precondition) + .await + } + + async fn finalize_snapshot_write( + &self, + snapshot: StrongTableCatalogSnapshot, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> { + match self.persist_snapshot(snapshot, precondition).await { + Ok(()) => self.reload_state_from_durable().await, + Err(err) => { + let _ = self.reload_state_from_durable().await; + Err(err) + } + } + } + + pub(super) async fn materialize_bucket_snapshot( + &self, + source: StrongTableCatalogBucketSnapshot, + ) -> TableCatalogStoreResult<(String, bool)> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + let table_bucket = source.table_bucket.table_bucket.clone(); + let (snapshot, precondition) = { + let state = self.state.lock().await; + if let Some(current) = Self::bucket_snapshot_from_state_locked(&state, &table_bucket) { + if current != source { + return Err(TableCatalogStoreError::Conflict(format!( + "durable strong catalog already contains different state for table bucket {table_bucket}" + ))); + } + let snapshot_etag = state + .snapshot_etag + .clone() + .ok_or_else(|| TableCatalogStoreError::Internal("durable strong catalog snapshot has no etag".to_string()))?; + return Ok((snapshot_etag, false)); + } + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + Self::insert_bucket_snapshot_locked(&mut draft_state, source)?; + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await?; + let state = self.state.lock().await; + let snapshot_etag = state + .snapshot_etag + .clone() + .ok_or_else(|| TableCatalogStoreError::Internal("durable strong catalog snapshot has no etag".to_string()))?; + Ok((snapshot_etag, true)) + } + + pub(super) async fn remove_bucket_snapshot_if_unchanged( + &self, + table_bucket: &str, + expected_fingerprint: &str, + ) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + let (snapshot, precondition) = { + let state = self.state.lock().await; + let Some(current) = Self::bucket_snapshot_from_state_locked(&state, table_bucket) else { + return Ok(()); + }; + if table_catalog_bucket_snapshot_fingerprint(¤t)? != expected_fingerprint { + return Err(TableCatalogStoreError::Conflict(format!( + "durable strong catalog state changed after materializing table bucket {table_bucket}" + ))); + } + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + Self::remove_bucket_from_state_locked(&mut draft_state, table_bucket); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await + } + + pub(super) async fn bucket_snapshot_fingerprint(&self, table_bucket: &str) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let state = self.state.lock().await; + Self::bucket_snapshot_from_state_locked(&state, table_bucket) + .as_ref() + .map(table_catalog_bucket_snapshot_fingerprint) + .transpose() + } + + fn require_table_bucket_in_state(state: &StrongTableCatalogState, table_bucket: &str) -> TableCatalogStoreResult<()> { + if !state.table_buckets.contains_key(table_bucket) { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + } + Ok(()) + } + + fn ensure_table_warehouse_prefix_available_locked( + state: &StrongTableCatalogState, + candidate: &TableEntry, + candidate_key: &StrongResourceKey, + ) -> TableCatalogStoreResult<()> { + if candidate.state != TableCatalogEntryState::Active { + return Ok(()); + } + let candidate_prefix = table_warehouse_object_prefix(candidate)?; + for (existing_key, existing) in &state.tables { + if existing_key == candidate_key + || existing.table_bucket != candidate.table_bucket + || existing.state != TableCatalogEntryState::Active + { + continue; + } + let Ok(existing_prefix) = table_warehouse_object_prefix(existing) else { + continue; + }; + if existing_prefix == candidate_prefix { + return Err(TableCatalogStoreError::Conflict(format!( + "table warehouse location is already registered: {candidate_prefix}" + ))); + } + } + Ok(()) + } + + fn table_commit_recovery_report_for_entry_locked( + state: &StrongTableCatalogState, + entry: &TableEntry, + ) -> TableCommitRecoveryReport { + let mut commits = state + .commits + .iter() + .filter(|((table_bucket, table_id, _), _)| table_bucket == &entry.table_bucket && table_id == &entry.table_id) + .map(|((_, _, _), commit_log)| { + let idempotency_commit = commit_log.idempotency_key.as_deref().and_then(|idempotency_key| { + state + .idempotency + .get(&Self::idempotency_key(&entry.table_bucket, &entry.table_id, idempotency_key)) + }); + table_commit_recovery_entry(entry, commit_log, idempotency_commit) + }) + .collect::>(); + commits.sort_by(|left, right| left.commit_id.cmp(&right.commit_id)); + + let finalization_required_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::FinalizationRequired)) + .count(); + let idempotency_repair_required_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::IdempotencyIndexRepairRequired)) + .count(); + let staged_before_table_update_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::StagedBeforeTableUpdate)) + .count(); + let manual_review_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::ManualReview)) + .count(); + let finalized_count = commits + .iter() + .filter(|commit| matches!(commit.recovery_state, TableCommitRecoveryState::Committed)) + .count(); + + TableCommitRecoveryReport { + table_bucket: entry.table_bucket.clone(), + namespace: entry.namespace.clone(), + table: entry.table.clone(), + table_id: entry.table_id.clone(), + current_metadata_location: entry.metadata_location.clone(), + current_version_token: entry.version_token.clone(), + current_generation: entry.generation, + commits, + staged_before_table_update_count, + finalization_required_count, + idempotency_repair_required_count, + manual_review_count, + finalized_count, + } + } + + fn validate_new_table_commit_locked( + state: &StrongTableCatalogState, + key: &StrongResourceKey, + request: &TableCommitRequest, + namespace: &Namespace, + table: &IdentifierSegment, + ) -> TableCatalogStoreResult { + let Some(current) = state.tables.get(key).cloned() else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + request.table_bucket, request.namespace, request.table + ))); + }; + let commit_key = Self::commit_key(&request.table_bucket, ¤t.table_id, &request.commit_id); + let existing_commit = state.commits.get(&commit_key); + let idempotency_key = request + .idempotency_key + .as_deref() + .map(|idempotency_key| Self::idempotency_key(&request.table_bucket, ¤t.table_id, idempotency_key)); + let existing_idempotency_commit = idempotency_key.as_ref().and_then(|key| state.idempotency.get(key)); + + if let Some(existing) = existing_commit { + if !commit_log_matches_request(existing, request, ¤t.table_id) { + return Err(TableCatalogStoreError::Conflict(format!( + "commit id already exists: {}", + request.commit_id + ))); + } + if matches!(existing.status, CommitLogStatus::Committed) || table_matches_committed_log(¤t, existing) { + return Ok(current); + } + return Err(TableCatalogStoreError::Conflict( + "existing commit record does not match current table state".to_string(), + )); + } + if let Some(existing) = existing_idempotency_commit + && !commit_log_matches_request(existing, request, ¤t.table_id) + { + return Err(TableCatalogStoreError::Conflict("idempotency key already exists".to_string())); + } + if existing_idempotency_commit.is_some() { + return Err(TableCatalogStoreError::Conflict( + "idempotency key exists without a recoverable commit record".to_string(), + )); + } + if current.version_token != request.expected_version_token { + return Err(TableCatalogStoreError::Conflict( + "current table version token does not match expected token".to_string(), + )); + } + if current.metadata_location != request.expected_metadata_location { + return Err(TableCatalogStoreError::Conflict( + "current table metadata location does not match expected location".to_string(), + )); + } + if !is_valid_table_metadata_location(namespace, table, &request.new_metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "new metadata location must be inside the table metadata directory".to_string(), + )); + } + Ok(current) + } + + fn committed_existing_result_locked( + state: &mut StrongTableCatalogState, + request: &TableCommitRequest, + current: TableEntry, + ) -> Option { + let commit_key = Self::commit_key(&request.table_bucket, ¤t.table_id, &request.commit_id); + let existing = state.commits.get(&commit_key)?; + if !commit_log_matches_request(existing, request, ¤t.table_id) { + return None; + } + if !matches!(existing.status, CommitLogStatus::Committed) && !table_matches_committed_log(¤t, existing) { + return None; + } + + let mut committed = existing.clone(); + committed.status = CommitLogStatus::Committed; + state.commits.insert(commit_key, committed.clone()); + if let Some(idempotency_key) = committed.idempotency_key.as_deref() { + state.idempotency.insert( + Self::idempotency_key(&request.table_bucket, ¤t.table_id, idempotency_key), + committed.clone(), + ); + } + Some(TableCommitResult { + table: current, + commit_log: committed, + }) + } + + fn apply_commit_locked( + state: &mut StrongTableCatalogState, + request: &TableCommitRequest, + namespace: &Namespace, + table: &IdentifierSegment, + next_warehouse_location: Option, + ) -> TableCatalogStoreResult { + let key = Self::table_key(&request.table_bucket, namespace, table); + let current = Self::validate_new_table_commit_locked(state, &key, request, namespace, table)?; + if let Some(result) = Self::committed_existing_result_locked(state, request, current.clone()) { + return Ok(result); + } + + let commit_log = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: request.commit_id.clone(), + idempotency_key: request.idempotency_key.clone(), + table_id: current.table_id.clone(), + operation: request.operation.clone(), + expected_version_token: request.expected_version_token.clone(), + new_version_token: format!("token-{}", Uuid::new_v4()), + previous_metadata_location: current.metadata_location.clone(), + new_metadata_location: request.new_metadata_location.clone(), + requirements: request.requirements.clone(), + status: CommitLogStatus::Committed, + writer: request.writer.clone(), + created_at: None, + updated_at: None, + }; + + let mut next = current; + next.metadata_location = commit_log.new_metadata_location.clone(); + if let Some(warehouse_location) = next_warehouse_location { + next.warehouse_location = warehouse_location; + } + Self::ensure_table_warehouse_prefix_available_locked(state, &next, &key)?; + next.version_token = commit_log.new_version_token.clone(); + next.generation = next.generation.saturating_add(1); + + let commit_key = Self::commit_key(&request.table_bucket, &next.table_id, &request.commit_id); + state.commits.insert(commit_key, commit_log.clone()); + if let Some(idempotency_key) = request.idempotency_key.as_deref() { + state.idempotency.insert( + Self::idempotency_key(&request.table_bucket, &next.table_id, idempotency_key), + commit_log.clone(), + ); + } + state.tables.insert(key, next.clone()); + + Ok(TableCommitResult { table: next, commit_log }) + } + + pub(crate) async fn plan_table_commit_recovery( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult { + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let key = Self::table_key(table_bucket, &namespace, &table); + let state = self.state.lock().await; + let Some(entry) = state.tables.get(&key) else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + Ok(Self::table_commit_recovery_report_for_entry_locked(&state, entry)) + } +} + +#[async_trait::async_trait] +impl TableCatalogStore for StrongTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let state = self.state.lock().await; + Ok(state.table_buckets.get(table_bucket).cloned()) + } + + async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + validate_catalog_entry_version("table bucket", entry.version)?; + if entry.table_bucket.is_empty() { + return Err(TableCatalogStoreError::Invalid("table bucket name cannot be empty".to_string())); + } + if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE { + return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string())); + } + + let (snapshot, precondition) = { + let state = self.state.lock().await; + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + draft_state.table_buckets.insert(entry.table_bucket.clone(), entry); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await + } + + async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + validate_catalog_entry_version("namespace", entry.version)?; + let namespace = parse_namespace_for_store(&entry.namespace)?; + let key = Self::namespace_key(&entry.table_bucket, &namespace); + let (snapshot, precondition) = { + let state = self.state.lock().await; + Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; + if state.namespaces.contains_key(&key) { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog object already exists: namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + draft_state.namespaces.insert(key, entry); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await + } + + async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let state = self.state.lock().await; + let mut entries = state + .namespaces + .iter() + .filter(|((bucket, _), _)| bucket == table_bucket) + .map(|(_, entry)| entry.clone()) + .collect::>(); + entries.sort_by(|left, right| left.namespace.cmp(&right.namespace)); + Ok(entries) + } + + async fn list_namespaces_page( + &self, + table_bucket: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let cursor = catalog_list_cursor(cursor, STRONG_CATALOG_LIST_CURSOR_PREFIX)?; + let cursor = cursor + .map(parse_namespace_for_store) + .transpose()? + .map(|namespace| namespace.public_name()); + let start = match cursor { + Some(cursor) => Bound::Excluded((table_bucket.to_string(), cursor)), + None => Bound::Included((table_bucket.to_string(), String::new())), + }; + let state = self.state.lock().await; + let entries = state + .namespaces + .range((start, Bound::Unbounded)) + .take_while(|((bucket, _), _)| bucket == table_bucket) + .take(limit.get().saturating_add(1)) + .map(|(_, entry)| entry.clone()) + .collect(); + Ok(finish_catalog_list_page(entries, limit, STRONG_CATALOG_LIST_CURSOR_PREFIX, |entry| { + &entry.namespace + })) + } + + async fn get_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let state = self.state.lock().await; + Ok(state.namespaces.get(&Self::namespace_key(table_bucket, &namespace)).cloned()) + } + + async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let key = Self::namespace_key(table_bucket, &namespace); + let (snapshot, precondition) = { + let state = self.state.lock().await; + if !state.namespaces.contains_key(&key) { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + table_bucket, + namespace.public_name() + ))); + } + if state + .tables + .keys() + .any(|(bucket, namespace_name, _)| bucket == table_bucket && namespace_name == &namespace.public_name()) + || state + .views + .keys() + .any(|(bucket, namespace_name, _)| bucket == table_bucket && namespace_name == &namespace.public_name()) + { + return Err(TableCatalogStoreError::Conflict(format!( + "namespace {}/{} is not empty", + table_bucket, + namespace.public_name() + ))); + } + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + draft_state.namespaces.remove(&key); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await + } + + async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { + self.register_table(entry).await + } + + async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + validate_catalog_entry_version("table", entry.version)?; + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + table_warehouse_object_prefix(&entry)?; + let key = Self::table_key(&entry.table_bucket, &namespace, &table); + let (snapshot, precondition) = { + let state = self.state.lock().await; + Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; + if !state + .namespaces + .contains_key(&Self::namespace_key(&entry.table_bucket, &namespace)) + { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + if state.tables.contains_key(&key) { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog object already exists: table {}/{}/{}", + entry.table_bucket, entry.namespace, entry.table + ))); + } + Self::ensure_table_warehouse_prefix_available_locked(&state, &entry, &key)?; + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + draft_state.tables.insert(key, entry); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await + } + + async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let state = self.state.lock().await; + let mut entries = state + .tables + .iter() + .filter(|((bucket, namespace_name, _), _)| bucket == table_bucket && namespace_name == &namespace.public_name()) + .map(|(_, entry)| entry.clone()) + .collect::>(); + entries.sort_by(|left, right| left.table.cmp(&right.table)); + Ok(entries) + } + + async fn list_tables_page( + &self, + table_bucket: &str, + namespace: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?.public_name(); + let cursor = catalog_list_cursor(cursor, STRONG_CATALOG_LIST_CURSOR_PREFIX)?; + let cursor = cursor + .map(parse_table_for_store) + .transpose()? + .map(|table| table.as_str().to_string()); + let start = match cursor { + Some(cursor) => Bound::Excluded((table_bucket.to_string(), namespace.clone(), cursor)), + None => Bound::Included((table_bucket.to_string(), namespace.clone(), String::new())), + }; + let state = self.state.lock().await; + let entries = state + .tables + .range((start, Bound::Unbounded)) + .take_while(|((bucket, entry_namespace, _), _)| bucket == table_bucket && entry_namespace == &namespace) + .take(limit.get().saturating_add(1)) + .map(|(_, entry)| entry.clone()) + .collect(); + Ok(finish_catalog_list_page(entries, limit, STRONG_CATALOG_LIST_CURSOR_PREFIX, |entry| { + &entry.table + })) + } + + async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let state = self.state.lock().await; + Ok(state.tables.get(&Self::table_key(table_bucket, &namespace, &table)).cloned()) + } + + async fn resolve_table_data_plane_resource( + &self, + table_bucket: &str, + object: &str, + ) -> TableCatalogStoreResult> { + if table_bucket.is_empty() || object.is_empty() { + return Ok(None); + } + + self.hydrate_state().await?; + let state = self.state.lock().await; + let Some(bucket_entry) = state.table_buckets.get(table_bucket) else { + return Ok(None); + }; + if bucket_entry.state != TableCatalogEntryState::Active { + return Ok(None); + } + + let Some(bucket_index) = state.warehouse_index.get(table_bucket) else { + return Ok(None); + }; + + for warehouse_object_prefix in warehouse_index_candidate_prefixes(object) { + if let Some(table_key) = bucket_index.get(warehouse_object_prefix) { + let Some(table) = state.tables.get(table_key) else { + continue; + }; + return Ok(Some(table_data_plane_resource_from_entry( + table.clone(), + warehouse_object_prefix.to_string(), + ))); + } + } + Ok(None) + } + + async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + let commit_started = Instant::now(); + record_table_commit_attempt(&request.operation); + let namespace = parse_namespace_for_store(&request.namespace)?; + let table = parse_table_for_store(&request.table)?; + let key = Self::table_key(&request.table_bucket, &namespace, &table); + + let committed_existing_result = { + let state = self.state.lock().await; + let current = Self::validate_new_table_commit_locked(&state, &key, &request, &namespace, &table); + match current { + Ok(current) => { + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + Self::committed_existing_result_locked(&mut draft_state, &request, current).map(|result| { + Self::snapshot_from_mutated_state_locked(&mut draft_state) + .map(|snapshot| (result, snapshot, precondition)) + }) + } + Err(error) => { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(error), + ); + } + } + }; + if let Some(prepared_result) = committed_existing_result { + let result = match prepared_result { + Ok((result, snapshot, precondition)) => { + self.finalize_snapshot_write(snapshot, precondition).await.map(|_| result) + } + Err(err) => Err(err), + }; + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + result, + ); + } + + let Some(new_metadata_object) = self + .object_backend + .read_object(&request.table_bucket, &request.new_metadata_location) + .await? + else { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::NotFound(format!( + "new metadata object {}", + request.new_metadata_location + ))), + ); + }; + let next_warehouse_location = + table_metadata_warehouse_location(&request.table_bucket, &request.new_metadata_location, &new_metadata_object)?; + + let cas_started = Instant::now(); + let prepared_result = { + let state = self.state.lock().await; + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + match Self::apply_commit_locked(&mut draft_state, &request, &namespace, &table, next_warehouse_location) { + Ok(result) => { + Self::snapshot_from_mutated_state_locked(&mut draft_state).map(|snapshot| (result, snapshot, precondition)) + } + Err(err) => Err(err), + } + }; + let result = match prepared_result { + Ok((result, snapshot, precondition)) => self.finalize_snapshot_write(snapshot, precondition).await.map(|_| result), + Err(err) => Err(err), + }; + let cas_result = result.as_ref().map(|_| ()).map_err(Clone::clone); + record_table_commit_cas_result(&request.operation, cas_started, &cas_result); + table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + result, + ) + } + + async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let key = Self::table_key(table_bucket, &namespace, &table); + let (snapshot, precondition) = { + let state = self.state.lock().await; + if !state.tables.contains_key(&key) { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + table_bucket, + namespace.public_name(), + table.as_str() + ))); + } + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + draft_state.tables.remove(&key); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await + } + + async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + validate_catalog_entry_version("view", entry.version)?; + validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; + let namespace = parse_namespace_for_store(&entry.namespace)?; + let view = parse_table_for_store(&entry.view)?; + let key = Self::table_key(&entry.table_bucket, &namespace, &view); + let (snapshot, precondition) = { + let state = self.state.lock().await; + Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; + if !state + .namespaces + .contains_key(&Self::namespace_key(&entry.table_bucket, &namespace)) + { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + if state.views.contains_key(&key) { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog object already exists: view {}/{}/{}", + entry.table_bucket, entry.namespace, entry.view + ))); + } + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + draft_state.views.insert(key, entry); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await + } + + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let state = self.state.lock().await; + let mut entries = state + .views + .iter() + .filter(|((bucket, namespace_name, _), _)| bucket == table_bucket && namespace_name == &namespace.public_name()) + .map(|(_, entry)| entry.clone()) + .collect::>(); + entries.sort_by(|left, right| left.view.cmp(&right.view)); + Ok(entries) + } + + async fn list_views_page( + &self, + table_bucket: &str, + namespace: &str, + cursor: Option<&str>, + limit: NonZeroUsize, + ) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?.public_name(); + let cursor = catalog_list_cursor(cursor, STRONG_CATALOG_LIST_CURSOR_PREFIX)?; + let cursor = cursor + .map(parse_table_for_store) + .transpose()? + .map(|view| view.as_str().to_string()); + let start = match cursor { + Some(cursor) => Bound::Excluded((table_bucket.to_string(), namespace.clone(), cursor)), + None => Bound::Included((table_bucket.to_string(), namespace.clone(), String::new())), + }; + let state = self.state.lock().await; + let entries = state + .views + .range((start, Bound::Unbounded)) + .take_while(|((bucket, entry_namespace, _), _)| bucket == table_bucket && entry_namespace == &namespace) + .take(limit.get().saturating_add(1)) + .map(|(_, entry)| entry.clone()) + .collect(); + Ok(finish_catalog_list_page(entries, limit, STRONG_CATALOG_LIST_CURSOR_PREFIX, |entry| { + &entry.view + })) + } + + async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let view = parse_table_for_store(view)?; + let state = self.state.lock().await; + Ok(state.views.get(&Self::table_key(table_bucket, &namespace, &view)).cloned()) + } + + async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(&request.namespace)?; + let view = parse_table_for_store(&request.view)?; + if !is_valid_view_metadata_location(&namespace, &view, &request.new_metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "new metadata location must be inside the view metadata directory".to_string(), + )); + } + let Some(new_metadata_object) = self + .object_backend + .read_object(&request.table_bucket, &request.new_metadata_location) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "new view metadata object {}", + request.new_metadata_location + ))); + }; + let next_warehouse_location = + view_metadata_warehouse_location(&request.table_bucket, &request.new_metadata_location, &new_metadata_object)?; + + let key = Self::table_key(&request.table_bucket, &namespace, &view); + let (snapshot, precondition, next) = { + let state = self.state.lock().await; + let Some(current) = state.views.get(&key).cloned() else { + return Err(TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + request.table_bucket, request.namespace, request.view + ))); + }; + if current.version_token != request.expected_version_token { + return Err(TableCatalogStoreError::Conflict( + "current view version token does not match expected token".to_string(), + )); + } + if current.metadata_location != request.expected_metadata_location { + return Err(TableCatalogStoreError::Conflict( + "current view metadata location does not match expected location".to_string(), + )); + } + + let mut next = current; + next.metadata_location = request.new_metadata_location; + if let Some(warehouse_location) = next_warehouse_location { + next.warehouse_location = warehouse_location; + } + next.version_token = format!("token-{}", Uuid::new_v4()); + next.generation = next.generation.saturating_add(1); + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + draft_state.views.insert(key, next.clone()); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition, next) + }; + self.finalize_snapshot_write(snapshot, precondition).await?; + Ok(ViewCommitResult { view: next }) + } + + async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + let namespace = parse_namespace_for_store(namespace)?; + let view = parse_table_for_store(view)?; + let key = Self::table_key(table_bucket, &namespace, &view); + let (snapshot, precondition) = { + let state = self.state.lock().await; + if !state.views.contains_key(&key) { + return Err(TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + table_bucket, + namespace.public_name(), + view.as_str() + ))); + } + let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + draft_state.views.remove(&key); + (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + }; + self.finalize_snapshot_write(snapshot, precondition).await + } + + async fn get_commit_by_id( + &self, + table_bucket: &str, + table_id: &str, + commit_id: &str, + ) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let state = self.state.lock().await; + Ok(state + .commits + .get(&Self::commit_key(table_bucket, table_id, commit_id)) + .cloned()) + } + + async fn get_commit_by_idempotency_key( + &self, + table_bucket: &str, + table_id: &str, + idempotency_key: &str, + ) -> TableCatalogStoreResult> { + self.hydrate_state().await?; + let state = self.state.lock().await; + Ok(state + .idempotency + .get(&Self::idempotency_key(table_bucket, table_id, idempotency_key)) + .cloned()) + } +} diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs new file mode 100644 index 000000000..740661da7 --- /dev/null +++ b/rustfs/src/table_catalog/tests.rs @@ -0,0 +1,8580 @@ +use super::identifier::{ + TableIdentifier, TablePathResolver, default_namespace_marker_path, default_table_current_pointer_path, + default_table_lifecycle_path, default_table_marker_path, default_table_root_prefix, is_valid_table_metadata_file_name, + namespace_name_from_marker_path, table_name_from_marker_path, validate_object_mutation, +}; +use super::*; +use datafusion::{ + arrow::{ + array::{Array, Int32Array, Int64Array}, + datatypes::{DataType, Field, Schema, SchemaRef}, + record_batch::RecordBatch, + }, + parquet::arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder}, +}; +use std::assert_matches; +use std::sync::Arc; + +#[test] +fn reserved_table_object_key_matches_exact_prefix_and_children_only() { + assert!(is_reserved_table_object_key(".rustfs-table")); + assert!(is_reserved_table_object_key(".rustfs-table/")); + assert!(is_reserved_table_object_key(".rustfs-table/metadata/current.json")); + + assert!(!is_reserved_table_object_key("")); + assert!(!is_reserved_table_object_key(".rustfs-table-other")); + assert!(!is_reserved_table_object_key("prefix/.rustfs-table/object")); + assert!(!is_reserved_table_object_key("user/.rustfs-table")); +} + +#[test] +fn object_mutation_guard_only_blocks_reserved_prefix_for_table_buckets() { + assert!(validate_object_mutation(false, ".rustfs-table/current.json").is_ok()); + assert_eq!( + validate_object_mutation(true, ".rustfs-table/current.json").unwrap_err(), + TableObjectMutationError::ReservedCatalogObject + ); + assert!(validate_object_mutation(true, ".rustfs-table-other/current.json").is_ok()); +} + +#[tokio::test] +async fn bucket_object_mutation_guard_fails_closed_for_reserved_prefix_when_bucket_metadata_is_unavailable() { + assert_eq!( + validate_bucket_object_mutation("missing-bucket", ".rustfs-table/current.json") + .await + .unwrap_err(), + TableObjectMutationError::ReservedCatalogObject + ); + assert!( + validate_bucket_object_mutation("missing-bucket", "ordinary/current.json") + .await + .is_ok() + ); +} + +#[test] +fn table_bucket_marker_json_uses_stable_catalog_defaults() { + let marker = serde_json::to_value(TableBucketMarker::default()).unwrap(); + + assert_eq!(marker["version"], TABLE_BUCKET_CONFIG_VERSION); + assert_eq!(marker["catalog_type"], TABLE_BUCKET_CATALOG_TYPE); + assert_eq!(marker["reserved_prefix"], TABLE_RESERVED_PREFIX); + assert!(!table_bucket_marker_json().unwrap().is_empty()); +} + +#[test] +fn catalog_entry_structures_serialize_stable_fields() { + use std::collections::BTreeMap; + + let bucket = TableBucketEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "analytics".to_string(), + catalog_type: TABLE_BUCKET_CATALOG_TYPE.to_string(), + warehouse_root: "s3://analytics/".to_string(), + state: TableCatalogEntryState::Active, + properties: BTreeMap::from([("owner".to_string(), "platform".to_string())]), + created_at: Some("2026-05-23T00:00:00Z".to_string()), + updated_at: Some("2026-05-23T00:00:00Z".to_string()), + }; + let namespace = NamespaceEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "analytics".to_string(), + namespace: "sales".to_string(), + namespace_id: "sales".to_string(), + state: TableCatalogEntryState::Active, + properties: BTreeMap::from([("purpose".to_string(), "orders".to_string())]), + created_at: Some("2026-05-23T00:00:00Z".to_string()), + updated_at: Some("2026-05-23T00:00:00Z".to_string()), + }; + let table = TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "analytics".to_string(), + namespace: "sales".to_string(), + table: "orders".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://analytics/tables/table-id".to_string(), + metadata_location: "s3://analytics/tables/table-id/metadata/v1.metadata.json".to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::from([("write.format.default".to_string(), "parquet".to_string())]), + created_at: Some("2026-05-23T00:00:00Z".to_string()), + updated_at: Some("2026-05-23T00:00:00Z".to_string()), + }; + let commit = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-id".to_string(), + idempotency_key: Some("client-request-id".to_string()), + table_id: "table-id".to_string(), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: "s3://analytics/tables/table-id/metadata/v1.metadata.json".to_string(), + new_metadata_location: "s3://analytics/tables/table-id/metadata/v2.metadata.json".to_string(), + requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], + status: CommitLogStatus::Committed, + writer: Some("pyiceberg/test".to_string()), + created_at: Some("2026-05-23T00:01:00Z".to_string()), + updated_at: Some("2026-05-23T00:01:00Z".to_string()), + }; + + let bucket_json = serde_json::to_value(&bucket).unwrap(); + let namespace_json = serde_json::to_value(&namespace).unwrap(); + let table_json = serde_json::to_value(&table).unwrap(); + let commit_json = serde_json::to_value(&commit).unwrap(); + + assert_eq!(bucket_json["state"], "ACTIVE"); + assert_eq!(bucket_json["properties"]["owner"], "platform"); + assert_eq!(namespace_json["namespace_id"], "sales"); + assert_eq!(table_json["version_token"], "token-v1"); + assert_eq!(table_json["generation"], 1); + assert_eq!(table_json["state"], "ACTIVE"); + assert_eq!(commit_json["status"], "COMMITTED"); + assert_eq!(commit_json["requirements"][0]["type"], "assert-table-uuid"); +} + +#[test] +fn catalog_entry_deserialization_rejects_unknown_fields() { + use std::collections::BTreeMap; + + let table = TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "analytics".to_string(), + namespace: "sales".to_string(), + table: "orders".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://analytics/tables/table-id".to_string(), + metadata_location: "s3://analytics/tables/table-id/metadata/v1.metadata.json".to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let mut value = serde_json::to_value(table).unwrap(); + value + .as_object_mut() + .unwrap() + .insert("unexpected".to_string(), serde_json::json!(true)); + + assert!(serde_json::from_value::(value).is_err()); +} + +#[test] +#[serial_test::serial] +fn table_catalog_backing_mode_defaults_to_object() { + temp_env::with_var_unset(ENV_TABLE_CATALOG_BACKING, || { + assert_eq!(TableCatalogBackingMode::from_env().unwrap(), TableCatalogBackingMode::ObjectBacked); + }); +} + +#[test] +#[serial_test::serial] +fn table_catalog_backing_mode_accepts_durable_strong_value() { + temp_env::with_var(ENV_TABLE_CATALOG_BACKING, Some(TABLE_CATALOG_BACKING_DURABLE_STRONG), || { + assert_eq!(TableCatalogBackingMode::from_env().unwrap(), TableCatalogBackingMode::DurableStrong); + }); +} + +#[test] +#[serial_test::serial] +fn table_catalog_backing_mode_rejects_unknown_value() { + temp_env::with_var(ENV_TABLE_CATALOG_BACKING, Some("memory"), || { + assert!(matches!( + TableCatalogBackingMode::from_env().unwrap_err(), + TableCatalogStoreError::Invalid(_) + )); + }); +} + +struct NoopTableCatalogStore; + +#[async_trait::async_trait] +impl TableCatalogStore for NoopTableCatalogStore { + async fn get_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn put_table_bucket(&self, _entry: TableBucketEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn create_namespace(&self, _entry: NamespaceEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn list_namespaces(&self, _table_bucket: &str) -> TableCatalogStoreResult> { + Ok(Vec::new()) + } + + async fn get_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn drop_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn create_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn register_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn list_tables(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { + Ok(Vec::new()) + } + + async fn load_table( + &self, + _table_bucket: &str, + _namespace: &str, + _table: &str, + ) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { + let table = TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: request.table_bucket, + namespace: request.namespace, + table: request.table, + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://analytics/tables/table-id".to_string(), + metadata_location: request.new_metadata_location.clone(), + version_token: "token-v2".to_string(), + generation: 2, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let commit_log = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: request.commit_id, + idempotency_key: request.idempotency_key, + table_id: table.table_id.clone(), + operation: request.operation, + expected_version_token: request.expected_version_token, + new_version_token: table.version_token.clone(), + previous_metadata_location: request.expected_metadata_location, + new_metadata_location: table.metadata_location.clone(), + requirements: request.requirements, + status: CommitLogStatus::Committed, + writer: request.writer, + created_at: None, + updated_at: None, + }; + + Ok(TableCommitResult { table, commit_log }) + } + + async fn drop_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn create_view(&self, _entry: ViewEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn list_views(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { + Ok(Vec::new()) + } + + async fn load_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + Ok(ViewCommitResult { + view: ViewEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: request.table_bucket, + namespace: request.namespace, + view: request.view, + view_id: "view-id".to_string(), + view_uuid: "view-uuid".to_string(), + format: "ICEBERG_VIEW".to_string(), + format_version: 1, + warehouse_location: "s3://analytics/views/view-id".to_string(), + metadata_location: request.new_metadata_location, + version_token: "token-v2".to_string(), + generation: 2, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }, + }) + } + + async fn drop_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn get_commit_by_id( + &self, + _table_bucket: &str, + _table_id: &str, + _commit_id: &str, + ) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn get_commit_by_idempotency_key( + &self, + _table_bucket: &str, + _table_id: &str, + _idempotency_key: &str, + ) -> TableCatalogStoreResult> { + Ok(None) + } +} + +#[tokio::test] +async fn table_catalog_store_trait_covers_entry_read_write_shapes() { + let store: &dyn TableCatalogStore = &NoopTableCatalogStore; + + assert!(store.get_table_bucket("analytics").await.unwrap().is_none()); + assert!(store.list_namespaces("analytics").await.unwrap().is_empty()); + assert!( + store + .get_commit_by_id("analytics", "table-id", "commit-id") + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_commit_by_idempotency_key("analytics", "table-id", "client-request-id") + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn table_catalog_store_trait_has_atomic_commit_shape() { + let store: &dyn TableCatalogStore = &NoopTableCatalogStore; + let request = TableCommitRequest { + table_bucket: "analytics".to_string(), + namespace: "sales".to_string(), + table: "orders".to_string(), + commit_id: "commit-id".to_string(), + idempotency_key: Some("client-request-id".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: "s3://analytics/tables/table-id/metadata/v1.metadata.json".to_string(), + new_metadata_location: "s3://analytics/tables/table-id/metadata/v2.metadata.json".to_string(), + requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], + writer: Some("pyiceberg/test".to_string()), + }; + + let result = store.commit_table(request).await.unwrap(); + + assert_eq!(result.table.version_token, "token-v2"); + assert_eq!(result.table.generation, 2); + assert_eq!(result.commit_log.status, CommitLogStatus::Committed); +} + +#[test] +fn catalog_object_entry_paths_use_internal_root_and_hashed_untrusted_ids() { + let paths = TableCatalogObjectPaths::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + let table = IdentifierSegment::parse("events").unwrap(); + let bucket_root = format!("s3tables/catalog/table-buckets/{}/", table_catalog_path_hash(bucket)); + + assert_eq!(paths.table_bucket_entry_path(bucket), format!("{bucket_root}table-bucket.json")); + assert_eq!( + paths.namespace_entry_path(bucket, &namespace), + format!("{bucket_root}namespaces/analytics/daily_events/namespace-entry.json") + ); + assert_eq!( + paths.table_entry_path(bucket, &namespace, &table), + format!("{bucket_root}namespaces/analytics/daily_events/tables/events/table-entry.json") + ); + assert_eq!( + paths.view_entry_path(bucket, &namespace, &table), + format!("{bucket_root}namespaces/analytics/daily_events/views/events/view-entry.json") + ); + + let commit_path = paths.commit_log_entry_path("table/../bucket", "table/../id", "commit/%2f\nid"); + let idempotency_path = paths.commit_idempotency_entry_path("table/../bucket", "table/../id", "client/%2f\nrequest"); + let warehouse_index_path = paths.warehouse_index_entry_path("table/../bucket", "tables/table/../id/data\nprefix/"); + let warehouse_index_state_path = paths.warehouse_index_state_path("table/../bucket"); + let maintenance_config_path = paths.table_maintenance_config_path("table/../bucket", &namespace, &table, "table/../id"); + let maintenance_job_path = + paths.table_maintenance_job_path("table/../bucket", &namespace, &table, "table/../id", "job/%2f\nid"); + + for path in [ + commit_path, + idempotency_path, + warehouse_index_path, + warehouse_index_state_path, + maintenance_config_path, + maintenance_job_path, + ] { + assert!(path.starts_with("s3tables/catalog/table-buckets/")); + assert!(path.ends_with(".json")); + assert!(!path.contains("..")); + assert!(!path.contains('%')); + assert!(!path.contains('\n')); + assert!(!path.contains("table/../bucket")); + assert!(!path.contains("table/../id")); + assert!(!path.contains("client/%2f")); + } +} + +#[derive(Clone, Default)] +struct TestCatalogObjectBackend { + state: Arc>, + locks: TestCatalogObjectLocks, +} + +type TestCatalogObjectLockKey = (String, String); +type TestCatalogObjectLock = Arc>; +type TestCatalogObjectLocks = Arc>>; + +#[derive(Clone, Default)] +struct TestCatalogObjectPutPause { + started: Arc, + release: Arc, +} + +impl TestCatalogObjectPutPause { + async fn wait_started(&self) { + self.started.notified().await; + } + + fn release(&self) { + self.release.notify_one(); + } +} + +#[derive(Default)] +struct TestCatalogObjectState { + objects: BTreeMap<(String, String), TestCatalogObjectRecord>, + fail_read_attempts: BTreeMap<(String, String), BTreeSet>, + read_attempts: BTreeMap<(String, String), usize>, + fail_put_attempts: BTreeMap<(String, String), BTreeSet>, + pause_put_attempts: BTreeMap<(String, String), BTreeMap>, + fail_delete_attempts: BTreeMap<(String, String), BTreeSet>, + put_attempts: BTreeMap<(String, String), usize>, + delete_attempts: BTreeMap<(String, String), usize>, + write_lock_acquisitions: BTreeMap<(String, String), usize>, + read_calls: usize, + metadata_calls: usize, + list_calls: usize, + next_etag: u64, +} + +#[derive(Clone)] +struct TestCatalogObjectRecord { + data: Vec, + etag: String, + mod_time: Option, +} + +impl TestCatalogObjectBackend { + async fn seed_object(&self, bucket: &str, object: &str, data: Vec) { + self.seed_object_with_mod_time(bucket, object, data, Some(OffsetDateTime::UNIX_EPOCH)) + .await; + } + + async fn seed_object_with_mod_time(&self, bucket: &str, object: &str, data: Vec, mod_time: Option) { + let mut state = self.state.lock().await; + let etag = state.next_etag(); + state + .objects + .insert((bucket.to_string(), object.to_string()), TestCatalogObjectRecord { data, etag, mod_time }); + } + + async fn fail_put_attempt(&self, bucket: &str, object: &str, attempt: usize) { + let mut state = self.state.lock().await; + state + .fail_put_attempts + .entry((bucket.to_string(), object.to_string())) + .or_default() + .insert(attempt); + } + + async fn fail_delete_attempt(&self, bucket: &str, object: &str, attempt: usize) { + let mut state = self.state.lock().await; + state + .fail_delete_attempts + .entry((bucket.to_string(), object.to_string())) + .or_default() + .insert(attempt); + } + + async fn list_call_count(&self) -> usize { + self.state.lock().await.list_calls + } + + async fn read_call_count(&self) -> usize { + self.state.lock().await.read_calls + } + + async fn metadata_call_count(&self) -> usize { + self.state.lock().await.metadata_calls + } + + async fn reset_call_counts(&self) { + let mut state = self.state.lock().await; + state.read_calls = 0; + state.metadata_calls = 0; + state.list_calls = 0; + } + + async fn write_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize { + self.state + .lock() + .await + .write_lock_acquisitions + .get(&(bucket.to_string(), object.to_string())) + .copied() + .unwrap_or_default() + } + + async fn fail_next_read(&self, bucket: &str, object: &str) { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1; + state.fail_read_attempts.entry(key).or_default().insert(next_attempt); + } + + async fn fail_next_put(&self, bucket: &str, object: &str) { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1; + state.fail_put_attempts.entry(key).or_default().insert(next_attempt); + } + + async fn pause_next_put(&self, bucket: &str, object: &str) -> TestCatalogObjectPutPause { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1; + let pause = TestCatalogObjectPutPause::default(); + state + .pause_put_attempts + .entry(key) + .or_default() + .insert(next_attempt, pause.clone()); + pause + } +} + +impl TestCatalogObjectState { + fn next_etag(&mut self) -> String { + self.next_etag += 1; + format!("etag-{}", self.next_etag) + } +} + +fn maintenance_object_report<'a>( + report: &'a TableMetadataMaintenanceReport, + metadata_location: &str, +) -> &'a TableMetadataMaintenanceObjectReport { + report + .object_reports + .iter() + .find(|object| object.metadata_location == metadata_location) + .expect("metadata maintenance object report should exist") +} + +fn snapshot_expiration_report( + report: &TableSnapshotExpirationReport, + snapshot_id: i64, +) -> &TableSnapshotExpirationSnapshotReport { + report + .snapshot_reports + .iter() + .find(|snapshot| snapshot.snapshot_id == Some(snapshot_id)) + .expect("snapshot expiration report should include the snapshot") +} + +fn compaction_snapshot_report(report: &TableCompactionPlanningReport, snapshot_id: i64) -> &TableCompactionSnapshotReport { + report + .snapshot_reports + .iter() + .find(|snapshot| snapshot.snapshot_id == Some(snapshot_id)) + .expect("compaction planning report should include the snapshot") +} + +fn object_cleanup_report<'a>( + report: &'a TableMetadataMaintenanceReport, + object_location: &str, +) -> &'a TableMetadataMaintenanceObjectCleanupReport { + report + .object_cleanup_reports + .iter() + .find(|object| object.object_location == object_location) + .expect("metadata maintenance object cleanup report should exist") +} + +fn manifest_list_avro_bytes(manifest_paths: &[&str]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string"}, + {"name": "partition_spec_id", "type": "int"}, + {"name": "sequence_number", "type": "long"}, + {"name": "added_snapshot_id", "type": "long"} + ] + } + "#, + ) + .expect("manifest list avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + for manifest_path in manifest_paths { + writer + .append(apache_avro::types::Value::Record(vec![ + ( + "manifest_path".to_string(), + apache_avro::types::Value::String((*manifest_path).to_string()), + ), + ("partition_spec_id".to_string(), apache_avro::types::Value::Int(0)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(7)), + ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(20)), + ])) + .expect("manifest list record should append"); + } + writer.into_inner().expect("manifest list avro bytes should flush") +} + +fn manifest_avro_bytes(files: &[(&str, i32)]) -> Vec { + manifest_avro_bytes_with_status( + &files + .iter() + .map(|(file_path, content)| (*file_path, *content, 1)) + .collect::>(), + ) +} + +fn manifest_avro_bytes_with_status(files: &[(&str, i32, i32)]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": "long"}, + {"name": "sequence_number", "type": "long"}, + {"name": "file_sequence_number", "type": "long"}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "data_file", + "fields": [ + {"name": "content", "type": "int"}, + {"name": "file_path", "type": "string"}, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"} + ] + } + } + ] + } + "#, + ) + .expect("manifest avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + for (file_path, content, status) in files { + writer + .append(apache_avro::types::Value::Record(vec![ + ("status".to_string(), apache_avro::types::Value::Int(*status)), + ("snapshot_id".to_string(), apache_avro::types::Value::Long(20)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(7)), + ("file_sequence_number".to_string(), apache_avro::types::Value::Long(7)), + ( + "data_file".to_string(), + apache_avro::types::Value::Record(vec![ + ("content".to_string(), apache_avro::types::Value::Int(*content)), + ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), + ("record_count".to_string(), apache_avro::types::Value::Long(1)), + ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), + ]), + ), + ])) + .expect("manifest record should append"); + } + writer.into_inner().expect("manifest avro bytes should flush") +} + +fn manifest_avro_bytes_with_dt_partition(files: &[(&str, i32, &str)]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": "long"}, + {"name": "sequence_number", "type": "long"}, + {"name": "file_sequence_number", "type": "long"}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "data_file", + "fields": [ + {"name": "content", "type": "int"}, + {"name": "file_path", "type": "string"}, + {"name": "partition", "type": {"type": "record", "name": "partition", "fields": [ + {"name": "dt", "type": "string"} + ]}}, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"} + ] + } + } + ] + } + "#, + ) + .expect("partitioned manifest avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + for (file_path, content, partition_value) in files { + writer + .append(apache_avro::types::Value::Record(vec![ + ("status".to_string(), apache_avro::types::Value::Int(1)), + ("snapshot_id".to_string(), apache_avro::types::Value::Long(20)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(7)), + ("file_sequence_number".to_string(), apache_avro::types::Value::Long(7)), + ( + "data_file".to_string(), + apache_avro::types::Value::Record(vec![ + ("content".to_string(), apache_avro::types::Value::Int(*content)), + ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), + ( + "partition".to_string(), + apache_avro::types::Value::Record(vec![( + "dt".to_string(), + apache_avro::types::Value::String((*partition_value).to_string()), + )]), + ), + ("record_count".to_string(), apache_avro::types::Value::Long(1)), + ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), + ]), + ), + ])) + .expect("partitioned manifest record should append"); + } + writer.into_inner().expect("partitioned manifest avro bytes should flush") +} + +fn manifest_avro_bytes_with_sort_order(files: &[(&str, i32, i32)]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": "long"}, + {"name": "sequence_number", "type": "long"}, + {"name": "file_sequence_number", "type": "long"}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "data_file", + "fields": [ + {"name": "content", "type": "int"}, + {"name": "file_path", "type": "string"}, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"}, + {"name": "sort_order_id", "type": ["null", "int"], "default": null} + ] + } + } + ] + } + "#, + ) + .expect("sort-order manifest avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + for (file_path, content, sort_order_id) in files { + writer + .append(apache_avro::types::Value::Record(vec![ + ("status".to_string(), apache_avro::types::Value::Int(1)), + ("snapshot_id".to_string(), apache_avro::types::Value::Long(20)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(7)), + ("file_sequence_number".to_string(), apache_avro::types::Value::Long(7)), + ( + "data_file".to_string(), + apache_avro::types::Value::Record(vec![ + ("content".to_string(), apache_avro::types::Value::Int(*content)), + ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), + ("record_count".to_string(), apache_avro::types::Value::Long(1)), + ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), + ( + "sort_order_id".to_string(), + apache_avro::types::Value::Union(1, Box::new(apache_avro::types::Value::Int(*sort_order_id))), + ), + ]), + ), + ])) + .expect("sort-order manifest record should append"); + } + writer.into_inner().expect("sort-order manifest avro bytes should flush") +} + +fn parquet_i32_bytes(values: &[i32]) -> Vec { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = RecordBatch::try_new(Arc::clone(&schema) as SchemaRef, vec![Arc::new(Int32Array::from(values.to_vec()))]) + .expect("parquet test batch should build"); + let mut bytes = Vec::new(); + { + let mut writer = ArrowWriter::try_new(&mut bytes, schema, None).expect("parquet writer should build"); + writer.write(&batch).expect("parquet batch should write"); + writer.close().expect("parquet writer should close"); + } + bytes +} + +fn parquet_i64_bytes(values: &[i64]) -> Vec { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new(Arc::clone(&schema) as SchemaRef, vec![Arc::new(Int64Array::from(values.to_vec()))]) + .expect("parquet test batch should build"); + let mut bytes = Vec::new(); + { + let mut writer = ArrowWriter::try_new(&mut bytes, schema, None).expect("parquet writer should build"); + writer.write(&batch).expect("parquet batch should write"); + writer.close().expect("parquet writer should close"); + } + bytes +} + +fn parquet_i32_values(data: Vec) -> Vec { + let reader = ParquetRecordBatchReaderBuilder::try_new(bytes::Bytes::from(data)) + .expect("parquet reader should build") + .build() + .expect("parquet batches should build"); + let mut values = Vec::new(); + for batch in reader { + let batch = batch.expect("parquet batch should read"); + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("first parquet column should be int32"); + values.extend((0..column.len()).map(|index| column.value(index))); + } + values +} + +#[async_trait::async_trait] +impl TableCatalogObjectBackend for TestCatalogObjectBackend { + async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + let mut state = self.state.lock().await; + state.read_calls += 1; + let key = (bucket.to_string(), object.to_string()); + let attempt = { + let attempts = state.read_attempts.entry(key.clone()).or_default(); + *attempts += 1; + *attempts + }; + if state + .fail_read_attempts + .get(&key) + .is_some_and(|attempts| attempts.contains(&attempt)) + { + return Err(TableCatalogStoreError::Internal(format!( + "injected read failure for {object} attempt {attempt}" + ))); + } + Ok(state.objects.get(&key).map(|record| TableCatalogObject { + data: record.data.clone(), + etag: Some(record.etag.clone()), + mod_time: record.mod_time, + })) + } + + async fn object_metadata(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + let mut state = self.state.lock().await; + state.metadata_calls += 1; + Ok(state + .objects + .get(&(bucket.to_string(), object.to_string())) + .map(|record| TableCatalogObjectMetadata { + etag: Some(record.etag.clone()), + mod_time: record.mod_time, + })) + } + + async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { + let state = self.state.lock().await; + Ok(state.objects.contains_key(&(bucket.to_string(), object.to_string()))) + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> { + let key = (bucket.to_string(), object.to_string()); + let pause = { + let mut state = self.state.lock().await; + let attempt = { + let attempts = state.put_attempts.entry(key.clone()).or_default(); + *attempts += 1; + *attempts + }; + if state + .fail_put_attempts + .get(&key) + .is_some_and(|attempts| attempts.contains(&attempt)) + { + return Err(TableCatalogStoreError::Internal(format!( + "injected put failure for {object} attempt {attempt}" + ))); + } + state + .pause_put_attempts + .get_mut(&key) + .and_then(|attempts| attempts.remove(&attempt)) + }; + if let Some(pause) = pause { + pause.started.notify_one(); + pause.release.notified().await; + } + + let mut state = self.state.lock().await; + match precondition { + TableCatalogPutPrecondition::IfAbsent if state.objects.contains_key(&key) => { + return Err(TableCatalogStoreError::Conflict(format!("object already exists: {object}"))); + } + TableCatalogPutPrecondition::IfMatch(expected) => { + let Some(current) = state.objects.get(&key) else { + return Err(TableCatalogStoreError::Conflict(format!("object is missing: {object}"))); + }; + if current.etag != expected { + return Err(TableCatalogStoreError::Conflict(format!("object changed: {object}"))); + } + } + _ => {} + } + + let etag = state.next_etag(); + state.objects.insert( + key, + TestCatalogObjectRecord { + data, + etag, + mod_time: Some(OffsetDateTime::now_utc()), + }, + ); + Ok(()) + } + + async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let attempt = { + let attempts = state.delete_attempts.entry(key.clone()).or_default(); + *attempts += 1; + *attempts + }; + if state + .fail_delete_attempts + .get(&key) + .is_some_and(|attempts| attempts.contains(&attempt)) + { + return Err(TableCatalogStoreError::Internal(format!( + "injected delete failure for {object} attempt {attempt}" + ))); + } + state.objects.remove(&key); + Ok(()) + } + + async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult> { + let mut state = self.state.lock().await; + state.list_calls += 1; + Ok(state + .objects + .keys() + .filter(|(entry_bucket, object)| entry_bucket == bucket && object.starts_with(prefix)) + .map(|(_, object)| object.clone()) + .collect()) + } + + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + { + let mut state = self.state.lock().await; + *state + .write_lock_acquisitions + .entry((bucket.to_string(), object.to_string())) + .or_default() += 1; + } + let lock = { + let mut locks = self.locks.lock().await; + locks + .entry((bucket.to_string(), object.to_string())) + .or_insert_with(|| std::sync::Arc::new(tokio::sync::Mutex::new(()))) + .clone() + }; + Ok(Box::new(lock.lock_owned().await)) + } +} + +fn test_bucket_entry(bucket: &str) -> TableBucketEntry { + TableBucketEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + catalog_type: TABLE_BUCKET_CATALOG_TYPE.to_string(), + warehouse_root: format!("s3://{bucket}/"), + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + } +} + +fn test_namespace_entry(bucket: &str, namespace: &Namespace) -> NamespaceEntry { + NamespaceEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + namespace_id: namespace.storage_id(), + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + } +} + +fn test_table_entry(bucket: &str, namespace: &Namespace, table: &IdentifierSegment, metadata_location: String) -> TableEntry { + TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: format!("s3://{bucket}/tables/table-id"), + metadata_location, + version_token: "token-v1".to_string(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + } +} + +fn test_view_entry(bucket: &str, namespace: &Namespace, view: &IdentifierSegment, metadata_location: String) -> ViewEntry { + ViewEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + view_id: "view-id".to_string(), + view_uuid: "view-uuid".to_string(), + format: "ICEBERG_VIEW".to_string(), + format_version: 1, + warehouse_location: format!("s3://{bucket}/views/view-id"), + metadata_location, + version_token: "token-v1".to_string(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + } +} + +async fn seed_catalog_list_entries(store: &S, bucket: &str, namespace: &Namespace) +where + S: TableCatalogStore + ?Sized, +{ + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + for namespace in [ + Namespace::parse("analytics").expect("namespace should parse"), + namespace.clone(), + ] { + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + } + for name in ["alpha", "beta"] { + let identifier = IdentifierSegment::parse(name).expect("table and view name should parse"); + let metadata_location = default_table_metadata_file_path(namespace, &identifier, "00001.metadata.json"); + let mut table = test_table_entry(bucket, namespace, &identifier, metadata_location.clone()); + table.table_id = format!("table-{name}"); + table.table_uuid = format!("table-uuid-{name}"); + table.warehouse_location = format!("s3://{bucket}/tables/table-{name}"); + store.create_table(table).await.expect("table should be created"); + + let mut view = test_view_entry(bucket, namespace, &identifier, metadata_location); + view.view_id = format!("view-{name}"); + view.view_uuid = format!("view-uuid-{name}"); + view.warehouse_location = format!("s3://{bucket}/views/view-{name}"); + store.create_view(view).await.expect("view should be created"); + } +} + +async fn seed_table_for_metadata_maintenance( + store: &ObjectTableCatalogStore, + bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + current_metadata: String, +) { + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store.create_namespace(test_namespace_entry(bucket, namespace)).await.unwrap(); + store + .create_table(test_table_entry(bucket, namespace, table, current_metadata)) + .await + .unwrap(); + store.backfill_table_warehouse_index(bucket).await.unwrap(); + store.backend.reset_call_counts().await; +} + +async fn seed_quarantined_table_maintenance( + store: &ObjectTableCatalogStore, + backend: &TestCatalogObjectBackend, + bucket: &str, + now: OffsetDateTime, + next_retry_after: Option, +) -> (Namespace, IdentifierSegment, TableMetadataMaintenanceReport) { + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + max_retry_attempts: 2, + retry_initial_backoff_seconds: 60, + retry_max_backoff_seconds: 300, + quarantine_enabled: true, + quarantine_retention_seconds: 86_400, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + let mut failed = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + failed.job.status = TableMetadataMaintenanceJobStatus::Failed; + failed.job.failure_reason = Some("quarantine retained failed cleanup candidates".to_string()); + failed.job.max_retry_attempts = 2; + failed.job.next_retry_after = next_retry_after.map(maintenance_timestamp); + failed.job.quarantine_enabled = true; + failed.job.quarantine_retention_seconds = 86_400; + failed.job.quarantined_object_count = 2; + failed.job.finished_at = Some(maintenance_timestamp(now - Duration::seconds(10))); + store + .put_table_metadata_maintenance_report(&failed) + .await + .expect("failed maintenance report should be seeded"); + (namespace, table, failed) +} + +#[tokio::test] +async fn object_table_catalog_store_writes_catalog_entries_to_internal_meta_bucket() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + assert!(!store.warehouse_index_ready(bucket).await.unwrap()); + + let state = backend.state.lock().await; + let object_buckets = state + .objects + .keys() + .map(|(bucket, _)| bucket.as_str()) + .collect::>(); + + assert_eq!(object_buckets, BTreeSet::from([RUSTFS_META_BUCKET])); +} + +#[tokio::test] +async fn configured_table_catalog_store_uses_durable_strong_snapshot() { + let backend = TestCatalogObjectBackend::default(); + let store = ConfiguredTableCatalogStore::new(backend.clone(), TableCatalogBackingMode::DurableStrong); + let bucket = "analytics"; + + assert_eq!(store.backing_mode(), TableCatalogBackingMode::DurableStrong); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + assert!(backend.object_exists(RUSTFS_META_BUCKET, &snapshot_path).await.unwrap()); + + let reloaded = ConfiguredTableCatalogStore::new(backend.clone(), TableCatalogBackingMode::DurableStrong); + assert!(reloaded.get_table_bucket(bucket).await.unwrap().is_some()); +} + +#[tokio::test] +async fn object_table_catalog_store_persists_view_entries_and_blocks_non_empty_namespace_drop() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let view = IdentifierSegment::parse("recent_orders").unwrap(); + let current_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let next_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_view(test_view_entry(bucket, &namespace, &view, current_metadata.clone())) + .await + .unwrap(); + + assert_eq!(store.list_views(bucket, &namespace.public_name()).await.unwrap()[0].view, "recent_orders"); + assert!( + store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .unwrap() + .is_some() + ); + assert!(matches!( + store.drop_namespace(bucket, &namespace.public_name()).await, + Err(TableCatalogStoreError::Conflict(_)) + )); + + backend + .seed_object( + bucket, + &next_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": format!("s3://{bucket}/views/view-id") + })) + .unwrap(), + ) + .await; + let result = store + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata.clone(), + }) + .await + .unwrap(); + + assert_eq!(result.view.metadata_location, next_metadata); + assert_eq!(result.view.generation, 2); + assert_ne!(result.view.version_token, "token-v1"); + + store + .drop_view(bucket, &namespace.public_name(), view.as_str()) + .await + .unwrap(); + assert!(store.list_views(bucket, &namespace.public_name()).await.unwrap().is_empty()); + store.drop_namespace(bucket, &namespace.public_name()).await.unwrap(); +} + +#[tokio::test] +async fn maintenance_dry_run_keeps_current_metadata() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let v1 = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let v2 = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &v1, b"{}".to_vec()).await; + backend.seed_object(bucket, &v2, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + + assert_eq!(report.current_metadata_location, current); + assert!(report.retained_metadata_locations.contains(&report.current_metadata_location)); + assert!(!report.cleanup_candidate_locations.contains(&report.current_metadata_location)); + assert_eq!(report.cleanup_candidate_locations, vec![v1, v2]); +} + +#[tokio::test] +async fn table_data_plane_resource_resolves_registered_warehouse_prefix() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + assert_eq!(backend.list_call_count().await, 0); + + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("data-plane resource lookup should succeed") + .expect("object should resolve to the registered table"); + + assert_eq!(resource.table_bucket, bucket); + assert_eq!(resource.namespace, "sales"); + assert_eq!(resource.table, "orders"); + assert_eq!(resource.table_id, "table-id"); + assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); + assert_eq!(resource.catalog_resource_object(), "namespaces/sales/tables/orders"); + assert_eq!(backend.list_call_count().await, 0); +} + +#[tokio::test] +async fn table_data_plane_resource_does_not_match_sibling_prefix() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id-other/data/part-00001.parquet") + .await + .expect("data-plane resource lookup should succeed"); + + assert!(resource.is_none()); + assert_eq!(backend.list_call_count().await, 0); +} + +#[tokio::test] +async fn table_data_plane_resource_prefers_longest_registered_warehouse_prefix() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let parent_table = IdentifierSegment::parse("orders").unwrap(); + let child_table = IdentifierSegment::parse("orders_child").unwrap(); + let current = default_table_metadata_file_path(&namespace, &parent_table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent_table, current.clone()).await; + let mut child_entry = test_table_entry(bucket, &namespace, &child_table, current); + child_entry.table_id = "table-id-child".to_string(); + child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); + store.create_table(child_entry).await.unwrap(); + + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/child/data/part-00001.parquet") + .await + .expect("data-plane resource lookup should succeed") + .expect("object should resolve to the child table"); + + assert_eq!(resource.table, "orders_child"); + assert_eq!(resource.table_id, "table-id-child"); + assert_eq!(resource.warehouse_object_prefix, "tables/table-id/child/"); + assert_eq!(resource.catalog_resource_object(), "namespaces/sales/tables/orders_child"); + assert_eq!(backend.list_call_count().await, 0); +} + +#[tokio::test] +async fn table_data_plane_resource_skips_stale_deeper_index_and_matches_parent() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + let stale_prefix = "tables/table-id/child/"; + let stale_index = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: "table-id".to_string(), + warehouse_object_prefix: stale_prefix.to_string(), + state: TableCatalogEntryState::Active, + }; + store + .write_entry( + store.catalog_bucket(), + &store.paths.warehouse_index_entry_path(bucket, stale_prefix), + &stale_index, + TableCatalogPutPrecondition::Any, + ) + .await + .unwrap(); + + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/child/data/part-00001.parquet") + .await + .expect("stale deeper index should not fail lookup") + .expect("parent table should still protect the object"); + + assert_eq!(resource.table, "orders"); + assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); + assert_eq!(backend.list_call_count().await, 0); +} + +#[tokio::test] +async fn object_table_catalog_store_rejects_duplicate_warehouse_prefix() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let first_table = IdentifierSegment::parse("orders").unwrap(); + let second_table = IdentifierSegment::parse("returns").unwrap(); + let current = default_table_metadata_file_path(&namespace, &first_table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &first_table, current.clone()).await; + let mut second_entry = test_table_entry(bucket, &namespace, &second_table, current); + second_entry.table_id = "second-table-id".to_string(); + second_entry.warehouse_location = format!("s3://{bucket}/tables/table-id"); + + let error = store.create_table(second_entry).await.unwrap_err(); + + assert!(matches!( + error, + TableCatalogStoreError::Conflict(message) if message.contains("warehouse location is already registered") + )); +} + +#[tokio::test] +async fn table_data_plane_resource_fails_closed_for_missing_indexed_table() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let prefix = "tables/missing-table/"; + let index = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: "orders".to_string(), + table_id: "missing-table-id".to_string(), + warehouse_object_prefix: prefix.to_string(), + state: TableCatalogEntryState::Active, + }; + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store.backfill_table_warehouse_index(bucket).await.unwrap(); + store + .write_entry( + store.catalog_bucket(), + &store.paths.warehouse_index_entry_path(bucket, prefix), + &index, + TableCatalogPutPrecondition::Any, + ) + .await + .unwrap(); + + let error = table_data_plane_resource_for_object(&store, bucket, "tables/missing-table/data/part-00001.parquet") + .await + .unwrap_err(); + + assert!(matches!( + error, + TableCatalogStoreError::Internal(message) if message.contains("referenced table entry is missing") + )); +} + +#[tokio::test] +async fn object_table_catalog_store_replaces_stale_warehouse_index_on_create() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let prefix = "tables/shared-table/"; + let index_path = store.paths.warehouse_index_entry_path(bucket, prefix); + let stale_index = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: "missing_orders".to_string(), + table_id: "missing-table-id".to_string(), + warehouse_object_prefix: prefix.to_string(), + state: TableCatalogEntryState::Active, + }; + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .write_entry(store.catalog_bucket(), &index_path, &stale_index, TableCatalogPutPrecondition::Any) + .await + .unwrap(); + + let mut entry = test_table_entry(bucket, &namespace, &table, current); + entry.warehouse_location = format!("s3://{bucket}/tables/shared-table"); + store + .create_table(entry) + .await + .expect("stale warehouse index should be repaired before reserving the prefix"); + + let (index, _) = store + .read_entry::(store.catalog_bucket(), &index_path) + .await + .unwrap() + .expect("repaired index should exist"); + assert_eq!(index.table, "orders"); + assert_eq!(index.table_id, "table-id"); +} + +#[tokio::test] +async fn table_data_plane_resource_falls_back_to_scan_without_index_state() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let bucket_entry = test_bucket_entry(bucket); + let namespace_entry = test_namespace_entry(bucket, &namespace); + let table_entry = test_table_entry(bucket, &namespace, &table, current); + + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_bucket_entry_path(bucket), + &bucket_entry, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("table bucket entry should be seeded"); + store + .write_entry( + store.catalog_bucket(), + &store.paths.namespace_entry_path(bucket, &namespace), + &namespace_entry, + TableCatalogPutPrecondition::Any, + ) + .await + .unwrap(); + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &table), + &table_entry, + TableCatalogPutPrecondition::Any, + ) + .await + .unwrap(); + + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("legacy catalog lookup should fall back to scanning") + .expect("legacy table entry should resolve"); + + assert_eq!(resource.table, "orders"); + assert!(backend.list_call_count().await > 0); + assert!(store.warehouse_index_ready(bucket).await.unwrap()); + + backend.reset_call_counts().await; + let indexed_resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00002.parquet") + .await + .expect("backfilled index lookup should succeed") + .expect("backfilled table entry should resolve"); + + assert_eq!(indexed_resource.table, "orders"); + assert_eq!(backend.list_call_count().await, 0); + assert!(backend.read_call_count().await <= 5); +} + +#[tokio::test] +async fn object_table_catalog_store_backfill_skips_table_deleted_after_listing() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let bucket_entry = test_bucket_entry(bucket); + let namespace_entry = test_namespace_entry(bucket, &namespace); + let table_entry = test_table_entry(bucket, &namespace, &table, current); + let table_path = store.paths.table_entry_path(bucket, &namespace, &table); + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_bucket_entry_path(bucket), + &bucket_entry, + TableCatalogPutPrecondition::Any, + ) + .await + .unwrap(); + store + .write_entry( + store.catalog_bucket(), + &store.paths.namespace_entry_path(bucket, &namespace), + &namespace_entry, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("namespace entry should be seeded"); + store + .write_entry(store.catalog_bucket(), &table_path, &table_entry, TableCatalogPutPrecondition::Any) + .await + .expect("table entry should be seeded without an index"); + + let listed = store + .list_tables(bucket, &namespace.public_name()) + .await + .expect("table listing should succeed"); + assert_eq!(listed.len(), 1); + backend + .delete_object(RUSTFS_META_BUCKET, &table_path) + .await + .expect("listed table entry should be deleted before backfill"); + + for table in listed { + store + .backfill_active_table_warehouse_index(&table.table_bucket, &table.namespace, &table.table) + .await + .expect("backfill should skip a table deleted after listing"); + } + + assert!( + store + .read_entry::(store.catalog_bucket(), &index_path) + .await + .expect("warehouse index lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn object_catalog_pagination_bounds_reads_and_covers_rest_resources() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let namespace_name = namespace.public_name(); + let one = NonZeroUsize::new(1).expect("page size should be non-zero"); + + seed_catalog_list_entries(&store, bucket, &namespace).await; + + let namespace_page = store + .list_namespaces_page(bucket, None, one) + .await + .expect("first namespace page should load"); + assert_eq!(namespace_page.entries[0].namespace, "analytics"); + assert!( + namespace_page + .next_cursor + .as_deref() + .is_some_and(|cursor| cursor.starts_with(OBJECT_CATALOG_LIST_CURSOR_PREFIX)) + ); + let namespace_page = store + .list_namespaces_page(bucket, namespace_page.next_cursor.as_deref(), one) + .await + .expect("second namespace page should load"); + assert_eq!(namespace_page.entries[0].namespace, "sales"); + assert!(namespace_page.next_cursor.is_none()); + + backend.reset_call_counts().await; + let table_page = store + .list_tables_page(bucket, &namespace_name, None, one) + .await + .expect("first table page should load"); + assert_eq!(table_page.entries[0].table, "alpha"); + assert_eq!(backend.read_call_count().await, 1); + let table_page = store + .list_tables_page(bucket, &namespace_name, table_page.next_cursor.as_deref(), one) + .await + .expect("second table page should load"); + assert_eq!(table_page.entries[0].table, "beta"); + assert!(table_page.next_cursor.is_none()); + + let view_page = store + .list_views_page(bucket, &namespace_name, None, one) + .await + .expect("first view page should load"); + assert_eq!(view_page.entries[0].view, "alpha"); + let view_page = store + .list_views_page(bucket, &namespace_name, view_page.next_cursor.as_deref(), one) + .await + .expect("second view page should load"); + assert_eq!(view_page.entries[0].view, "beta"); + assert!(view_page.next_cursor.is_none()); + + let exact_page = store + .list_tables_page(bucket, &namespace_name, None, NonZeroUsize::new(2).expect("page size should be non-zero")) + .await + .expect("exact table page should load"); + assert_eq!(exact_page.entries.len(), 2); + assert!(exact_page.next_cursor.is_none()); + assert!(matches!( + store + .list_tables_page(bucket, &namespace_name, Some("strong:alpha"), one) + .await, + Err(TableCatalogStoreError::Invalid(_)) + )); +} + +#[tokio::test] +async fn object_catalog_pagination_bounds_sparse_namespace_scans() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let prefix = store.paths.namespace_entries_prefix(bucket); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + for index in 0..TABLE_CATALOG_LIST_MAX_KEYS { + backend + .seed_object(RUSTFS_META_BUCKET, &format!("{prefix}0000-spacer/{index:04}.json"), Vec::new()) + .await; + } + backend.reset_call_counts().await; + + let one = NonZeroUsize::new(1).expect("page size should be non-zero"); + let first = store + .list_namespaces_page(bucket, None, one) + .await + .expect("sparse first page should load"); + assert!(first.entries.is_empty()); + assert_eq!(backend.list_call_count().await, 1); + let cursor = first.next_cursor.expect("truncated sparse page should have a cursor"); + assert!(cursor.starts_with(OBJECT_CATALOG_LIST_CURSOR_PREFIX)); + assert!(!cursor.ends_with(NAMESPACE_ENTRY_FILE)); + + let second = store + .list_namespaces_page(bucket, Some(&cursor), one) + .await + .expect("sparse continuation page should load"); + assert_eq!(second.entries.len(), 1); + assert_eq!(second.entries[0].namespace, namespace.public_name()); + assert!(second.next_cursor.is_none()); + assert_eq!(backend.list_call_count().await, 2); +} + +#[tokio::test] +async fn object_table_catalog_store_rolls_back_warehouse_index_when_table_entry_write_fails() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let failed_table = IdentifierSegment::parse("failed_orders").unwrap(); + let next_table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &failed_table, "00001.metadata.json"); + let failed_table_path = store.paths.table_entry_path(bucket, &namespace, &failed_table); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + backend.fail_put_attempt(RUSTFS_META_BUCKET, &failed_table_path, 1).await; + + let mut failed_entry = test_table_entry(bucket, &namespace, &failed_table, current.clone()); + failed_entry.table_id = "failed-table-id".to_string(); + failed_entry.warehouse_location = format!("s3://{bucket}/tables/shared-table"); + let error = store.create_table(failed_entry).await.unwrap_err(); + assert!(matches!(error, TableCatalogStoreError::Internal(_))); + + let mut next_entry = test_table_entry(bucket, &namespace, &next_table, current); + next_entry.table_id = "next-table-id".to_string(); + next_entry.warehouse_location = format!("s3://{bucket}/tables/shared-table"); + store + .create_table(next_entry) + .await + .expect("rolled back warehouse index should not block the next table"); +} + +#[tokio::test] +async fn object_table_catalog_store_restores_table_when_drop_index_delete_fails() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend.fail_delete_attempt(RUSTFS_META_BUCKET, &index_path, 1).await; + + let error = store + .drop_table(bucket, &namespace.public_name(), table.as_str()) + .await + .unwrap_err(); + + assert!(matches!(error, TableCatalogStoreError::Internal(_))); + let restored = store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("restored table lookup should succeed") + .expect("table entry should be restored"); + assert_eq!(restored.table_id, "table-id"); + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("restored table data-plane lookup should succeed") + .expect("restored table should keep data-plane protection"); + assert_eq!(resource.table, "orders"); +} + +#[tokio::test] +async fn table_data_plane_resource_bounds_deep_object_index_reads() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend.reset_call_counts().await; + + let deep_suffix = (0..100).map(|index| format!("level-{index}/")).collect::(); + let object = format!("tables/table-id/{deep_suffix}part-00001.parquet"); + let resource = table_data_plane_resource_for_object(&store, bucket, &object) + .await + .expect("deep object lookup should succeed") + .expect("deep object should resolve to the table"); + + assert_eq!(resource.table, "orders"); + assert_eq!(backend.list_call_count().await, 0); + assert!(backend.read_call_count().await <= WAREHOUSE_INDEX_MAX_PREFIX_DEPTH + 3); +} + +#[tokio::test] +async fn object_table_catalog_store_rejects_invalid_table_warehouse_location() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + + let mut entry = test_table_entry(bucket, &namespace, &table, current); + entry.warehouse_location = format!("s3://{bucket}/tables/../table-id"); + + let error = store.create_table(entry).await.unwrap_err(); + assert!(matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("invalid path segment") + )); +} + +#[tokio::test] +async fn object_table_catalog_store_rejects_deep_table_warehouse_location() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let deep_prefix = (0..=WAREHOUSE_INDEX_MAX_PREFIX_DEPTH) + .map(|index| format!("level-{index}")) + .collect::>() + .join("/"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + + let mut entry = test_table_entry(bucket, &namespace, &table, current); + entry.warehouse_location = format!("s3://{bucket}/{deep_prefix}"); + + let error = store.create_table(entry).await.unwrap_err(); + assert!(matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("maximum prefix depth") + )); +} + +#[tokio::test] +async fn object_table_catalog_store_rejects_invalid_view_warehouse_location() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let current = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + + let mut entry = test_view_entry(bucket, &namespace, &view, current); + entry.warehouse_location = format!("s3://{bucket}/views/../view-id"); + + let error = store.create_view(entry).await.unwrap_err(); + assert!(matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("invalid path segment") + )); +} + +#[tokio::test] +async fn object_table_catalog_store_allows_deep_view_warehouse_location() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let view = IdentifierSegment::parse("recent_orders").unwrap(); + let current = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let next = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + let deep_prefix = (0..=WAREHOUSE_INDEX_MAX_PREFIX_DEPTH) + .map(|index| format!("level-{index}")) + .collect::>() + .join("/"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + + let mut entry = test_view_entry(bucket, &namespace, &view, current.clone()); + entry.warehouse_location = format!("s3://{bucket}/{deep_prefix}"); + store + .create_view(entry) + .await + .expect("view warehouse location should not inherit table index depth limits"); + + let relocated_prefix = format!("{deep_prefix}/relocated"); + backend + .seed_object( + bucket, + &next, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": format!("s3://{bucket}/{relocated_prefix}") + })) + .expect("view metadata should serialize"), + ) + .await; + + let result = store + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current, + new_metadata_location: next, + }) + .await + .expect("view metadata location should not inherit table index depth limits"); + + assert_eq!(result.view.warehouse_location, format!("s3://{bucket}/{relocated_prefix}")); +} + +#[tokio::test] +async fn table_data_plane_resource_skips_invalid_warehouse_locations() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let invalid_table = IdentifierSegment::parse("bad_orders").unwrap(); + let valid_table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &valid_table, "00001.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + + let mut invalid_entry = test_table_entry(bucket, &namespace, &invalid_table, current.clone()); + invalid_entry.table_id = "bad-table-id".to_string(); + invalid_entry.warehouse_location = format!("s3://{bucket}/"); + let invalid_path = store.paths.table_entry_path(bucket, &namespace, &invalid_table); + store + .write_entry( + store.catalog_bucket(), + &invalid_path, + &invalid_entry, + TableCatalogPutPrecondition::IfAbsent, + ) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &valid_table, current)) + .await + .unwrap(); + + let unrelated = table_data_plane_resource_for_object(&store, bucket, "ordinary/object.parquet") + .await + .expect("invalid table warehouse location should not deny unrelated object lookup"); + assert!(unrelated.is_none()); + + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("invalid table warehouse location should not block a later valid match") + .expect("valid table warehouse object should resolve to the table"); + assert_eq!(resource.table, "orders"); + assert_eq!(resource.table_id, "table-id"); + assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); +} + +#[tokio::test] +async fn maintenance_dry_run_reports_job_context_and_deletable_candidates() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let fresh = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + backend + .seed_object_with_mod_time(bucket, &fresh, b"{}".to_vec(), Some(OffsetDateTime::now_utc())) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + + assert_eq!(report.job.table_bucket, bucket); + assert_eq!(report.job.namespace, "sales"); + assert_eq!(report.job.table, "orders"); + assert_eq!(report.job.table_id, "table-id"); + assert_eq!(report.job.operation, TableMetadataMaintenanceOperation::DryRun); + assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Successful); + assert_eq!(report.job.deleted_metadata_file_count, 0); + assert_eq!(report.job.current_generation, 1); + assert_eq!(report.job.safety_window_seconds, TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS); + assert!(!report.job.job_id.is_empty()); + assert!(report.job.cleanup_watermark_unix_seconds <= OffsetDateTime::now_utc().unix_timestamp()); + assert_eq!(report.cleanup_candidate_locations, vec![old.clone(), fresh]); + assert_eq!(report.deletable_metadata_locations, vec![old]); +} + +#[tokio::test] +async fn maintenance_dry_run_explains_metadata_reachability() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let logged = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let fresh = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let old = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + let recent = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00005.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &logged, b"{}".to_vec()).await; + backend.seed_object(bucket, &recent, b"{}".to_vec()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object_with_mod_time(bucket, &fresh, b"{}".to_vec(), Some(OffsetDateTime::now_utc())) + .await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [ + { + "timestamp-ms": 1, + "metadata-file": logged + } + ] + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 2) + .await + .unwrap(); + + assert_eq!(report.job.planned_metadata_file_count, 5); + assert_eq!(report.job.retained_metadata_file_count, 3); + assert_eq!(report.job.cleanup_candidate_count, 2); + assert_eq!(report.job.deletable_metadata_file_count, 1); + assert_eq!( + report.job.recommended_actions, + vec![TableMaintenanceRecommendedAction::ReviewAndRunDelete] + ); + + let current_report = maintenance_object_report(&report, ¤t); + assert_eq!(current_report.state, TableMetadataMaintenanceObjectState::Retained); + assert_eq!(current_report.reasons, vec![TableMetadataMaintenanceReason::CurrentMetadata]); + + let logged_report = maintenance_object_report(&report, &logged); + assert_eq!(logged_report.state, TableMetadataMaintenanceObjectState::Retained); + assert_eq!(logged_report.reasons, vec![TableMetadataMaintenanceReason::MetadataLog]); + + let recent_report = maintenance_object_report(&report, &recent); + assert_eq!(recent_report.state, TableMetadataMaintenanceObjectState::Retained); + assert_eq!(recent_report.reasons, vec![TableMetadataMaintenanceReason::RecentMetadata]); + + let old_report = maintenance_object_report(&report, &old); + assert_eq!(old_report.state, TableMetadataMaintenanceObjectState::Deletable); + assert_eq!( + old_report.reasons, + vec![ + TableMetadataMaintenanceReason::NoCurrentReachability, + TableMetadataMaintenanceReason::SafetyWindowSatisfied, + ] + ); + + let fresh_report = maintenance_object_report(&report, &fresh); + assert_eq!(fresh_report.state, TableMetadataMaintenanceObjectState::PendingSafetyWindow); + assert_eq!( + fresh_report.reasons, + vec![ + TableMetadataMaintenanceReason::NoCurrentReachability, + TableMetadataMaintenanceReason::SafetyWindowPending, + ] + ); +} + +#[tokio::test] +async fn maintenance_report_read_back_derives_actions_for_legacy_records() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + let mut report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + report.job.status = TableMetadataMaintenanceJobStatus::Running; + report.job.worker_id = Some("worker-a".to_string()); + report.job.lease_id = "lease-a".to_string(); + report.job.heartbeat_at = Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10))); + + let job_path = store + .paths + .table_maintenance_job_path(bucket, &namespace, &table, "table-id", &report.job.job_id); + let mut legacy_report = serde_json::to_value(&report).expect("legacy report should serialize"); + legacy_report + .get_mut("job") + .and_then(serde_json::Value::as_object_mut) + .expect("legacy report job should be an object") + .remove("recommended-actions"); + store + .write_entry(store.catalog_bucket(), &job_path, &legacy_report, TableCatalogPutPrecondition::Any) + .await + .expect("legacy maintenance report should be seeded"); + + let loaded = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id) + .await + .expect("legacy maintenance report lookup should succeed") + .expect("legacy maintenance report should be returned"); + + assert_eq!( + loaded.job.recommended_actions, + vec![TableMaintenanceRecommendedAction::WaitForActiveWorker] + ); +} + +#[tokio::test] +async fn maintenance_state_is_scoped_to_current_table_identity() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + + let mut first_table = test_table_entry(bucket, &namespace, &table, current.clone()); + first_table.table_id = "table-id-1".to_string(); + store.create_table(first_table).await.unwrap(); + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 7, + delete_enabled: true, + background_enabled: false, + ..Default::default() + }, + ) + .await + .unwrap(); + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + store.put_table_metadata_maintenance_report(&report).await.unwrap(); + assert!( + store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id) + .await + .unwrap() + .is_some() + ); + + store.drop_table(bucket, "sales", "orders").await.unwrap(); + + let mut second_table = test_table_entry(bucket, &namespace, &table, current); + second_table.table_id = "table-id-2".to_string(); + store.create_table(second_table).await.unwrap(); + + assert_eq!( + store.get_table_maintenance_config(bucket, "sales", "orders").await.unwrap(), + TableMaintenanceConfig::default() + ); + assert!( + store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn maintenance_config_rejects_unsupported_config_version() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + + let err = store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION.saturating_add(1), + retain_recent_metadata_files: 1, + delete_enabled: false, + background_enabled: false, + ..Default::default() + }, + ) + .await + .unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Invalid(_)); +} + +#[tokio::test] +async fn maintenance_config_inherits_bucket_default_and_tracks_override_source() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + store + .put_table_bucket_maintenance_config( + bucket, + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 3, + delete_enabled: true, + background_enabled: false, + ..Default::default() + }, + ) + .await + .expect("bucket default maintenance config should persist"); + + let inherited = store + .get_effective_table_maintenance_config(bucket, "sales", "orders") + .await + .expect("effective maintenance config should load"); + + assert_eq!(inherited.source, TableMaintenanceConfigSource::TableBucketDefault); + assert_eq!(inherited.config.retain_recent_metadata_files, 3); + assert!(inherited.config.delete_enabled); + assert!(!inherited.config.background_enabled); + + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 1, + delete_enabled: false, + background_enabled: false, + ..Default::default() + }, + ) + .await + .expect("table maintenance override should persist"); + + let overridden = store + .get_effective_table_maintenance_config(bucket, "sales", "orders") + .await + .expect("effective maintenance override should load"); + + assert_eq!(overridden.source, TableMaintenanceConfigSource::TableOverride); + assert_eq!(overridden.config.retain_recent_metadata_files, 1); + assert!(!overridden.config.delete_enabled); + assert!(!overridden.config.background_enabled); +} + +#[tokio::test] +async fn maintenance_config_accepts_background_enabled_worker_runtime_controls() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + + let bucket_config = store + .put_table_bucket_maintenance_config( + bucket, + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 1, + delete_enabled: false, + background_enabled: true, + worker_paused: true, + worker_lease_timeout_seconds: 60, + ..Default::default() + }, + ) + .await + .expect("background maintenance bucket config should persist"); + assert!(bucket_config.background_enabled); + assert!(bucket_config.worker_paused); + assert_eq!(bucket_config.worker_lease_timeout_seconds, 60); + + let table_config = store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 1, + delete_enabled: false, + background_enabled: true, + worker_paused: false, + worker_lease_timeout_seconds: 120, + ..Default::default() + }, + ) + .await + .expect("background maintenance table config should persist"); + assert!(table_config.background_enabled); + assert!(!table_config.worker_paused); + assert_eq!(table_config.worker_lease_timeout_seconds, 120); +} + +#[tokio::test] +async fn maintenance_config_accepts_retry_and_quarantine_policy() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + + let config = store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 2, + delete_enabled: false, + background_enabled: false, + max_retry_attempts: 3, + retry_initial_backoff_seconds: 10, + retry_max_backoff_seconds: 60, + quarantine_enabled: true, + quarantine_retention_seconds: 86_400, + ..Default::default() + }, + ) + .await + .expect("retry and quarantine maintenance config should persist"); + + assert_eq!(config.max_retry_attempts, 3); + assert_eq!(config.retry_initial_backoff_seconds, 10); + assert_eq!(config.retry_max_backoff_seconds, 60); + assert!(config.quarantine_enabled); + assert_eq!(config.quarantine_retention_seconds, 86_400); +} + +#[tokio::test] +async fn maintenance_config_rejects_retry_backoff_above_limit() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + + let initial_err = store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 1, + delete_enabled: false, + background_enabled: false, + max_retry_attempts: 1, + retry_initial_backoff_seconds: TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS.saturating_add(1), + retry_max_backoff_seconds: TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS.saturating_add(1), + ..Default::default() + }, + ) + .await + .unwrap_err(); + assert_matches!(initial_err, TableCatalogStoreError::Invalid(_)); + + let max_err = store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 1, + delete_enabled: false, + background_enabled: false, + max_retry_attempts: 1, + retry_initial_backoff_seconds: TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS, + retry_max_backoff_seconds: TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS.saturating_add(1), + ..Default::default() + }, + ) + .await + .unwrap_err(); + assert_matches!(max_err, TableCatalogStoreError::Invalid(_)); +} + +#[tokio::test] +async fn maintenance_run_rejects_existing_invalid_retry_config_before_scheduling() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + let config_path = store + .paths + .table_maintenance_config_path(bucket, &namespace, &table, "table-id"); + store + .write_entry( + store.catalog_bucket(), + &config_path, + &TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 1, + delete_enabled: false, + background_enabled: false, + max_retry_attempts: 1, + retry_initial_backoff_seconds: u64::MAX, + retry_max_backoff_seconds: u64::MAX, + ..Default::default() + }, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("invalid legacy maintenance config should be seeded"); + + let err = store + .run_table_metadata_maintenance(bucket, "sales", "orders", true, Some("worker-a".to_string())) + .await + .unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Invalid(_)); +} + +#[tokio::test] +async fn maintenance_run_persists_latest_job_alias_with_worker_and_lease_context() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + store + .put_table_bucket_maintenance_config( + bucket, + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 0, + delete_enabled: false, + background_enabled: false, + ..Default::default() + }, + ) + .await + .expect("bucket default maintenance config should persist"); + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let report = store + .run_table_metadata_maintenance(bucket, "sales", "orders", false, Some("worker-a".to_string())) + .await + .expect("metadata maintenance run should succeed"); + + assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Successful); + assert_eq!(report.job.config_source, TableMaintenanceConfigSource::TableBucketDefault); + assert_eq!(report.job.worker_id.as_deref(), Some("worker-a")); + assert!(!report.job.lease_id.is_empty()); + assert_eq!(report.job.attempt, 1); + assert_eq!(report.job.max_retry_attempts, 0); + assert!(report.job.next_retry_after.is_none()); + assert!(report.job.heartbeat_at.is_some()); + assert!(report.job.started_at.is_some()); + assert!(report.job.finished_at.is_some()); + + let latest = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", "latest") + .await + .expect("latest maintenance lookup should succeed") + .expect("latest maintenance job should be stored"); + let current_alias = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", "current") + .await + .expect("current maintenance lookup should succeed") + .expect("current maintenance job should be stored"); + + assert_eq!(latest.job.job_id, report.job.job_id); + assert_eq!(current_alias.job.job_id, report.job.job_id); +} + +#[tokio::test] +async fn maintenance_delete_request_records_failed_job_when_delete_is_disabled() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 0, + delete_enabled: false, + background_enabled: false, + max_retry_attempts: 2, + retry_initial_backoff_seconds: 10, + retry_max_backoff_seconds: 30, + quarantine_enabled: true, + quarantine_retention_seconds: 86_400, + ..Default::default() + }, + ) + .await + .expect("table maintenance override should persist"); + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let report = store + .run_table_metadata_maintenance(bucket, "sales", "orders", true, Some("worker-a".to_string())) + .await + .expect("disabled delete request should still persist a failed maintenance job"); + + assert_eq!(report.job.operation, TableMetadataMaintenanceOperation::Delete); + assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Failed); + assert_eq!(report.job.config_source, TableMaintenanceConfigSource::TableOverride); + assert_eq!(report.job.max_retry_attempts, 2); + assert!(report.job.next_retry_after.is_some()); + assert!(report.job.quarantine_enabled); + assert_eq!(report.job.quarantine_retention_seconds, 86_400); + assert!( + report + .job + .failure_reason + .as_deref() + .is_some_and(|reason| reason.contains("disabled")) + ); + assert_eq!( + report.job.recommended_actions, + vec![ + TableMaintenanceRecommendedAction::EnableDelete, + TableMaintenanceRecommendedAction::WaitForRetryBackoff, + ] + ); + assert!(backend.object_exists(bucket, &old).await.unwrap()); + + let latest = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", "latest") + .await + .expect("latest maintenance lookup should succeed") + .expect("failed maintenance job should be stored"); + assert_eq!(latest.job.job_id, report.job.job_id); + assert_eq!(latest.job.status, TableMetadataMaintenanceJobStatus::Failed); + assert_eq!(latest.job.recommended_actions, report.job.recommended_actions); +} + +#[tokio::test] +async fn maintenance_worker_run_skips_when_background_is_disabled() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let table_path = store.paths.table_entry_path(bucket, &namespace, &table); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + let before_worker_run = backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &table_path).await; + + let report = store + .run_table_metadata_maintenance_worker_once(bucket, "sales", "orders", "worker-a".to_string()) + .await + .expect("disabled background worker tick should report a safe no-op"); + + assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Disabled); + assert_eq!(report.job.worker_id.as_deref(), Some("worker-a")); + assert_eq!( + report.job.recommended_actions, + vec![TableMaintenanceRecommendedAction::EnableBackgroundMaintenance] + ); + assert_eq!(report.job.deleted_metadata_file_count, 0); + assert_eq!( + backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &table_path).await, + before_worker_run + 1 + ); + assert!(backend.object_exists(bucket, &old).await.unwrap()); +} + +#[tokio::test] +async fn maintenance_worker_run_honors_paused_config() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 0, + delete_enabled: true, + background_enabled: true, + worker_paused: true, + ..Default::default() + }, + ) + .await + .expect("paused background maintenance config should persist"); + + let report = store + .run_table_metadata_maintenance_worker_once(bucket, "sales", "orders", "worker-a".to_string()) + .await + .expect("paused worker tick should report a safe no-op"); + + assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Paused); + assert_eq!(report.job.operation, TableMetadataMaintenanceOperation::DryRun); + assert_eq!( + report.job.recommended_actions, + vec![TableMaintenanceRecommendedAction::ResumeMaintenanceWorker] + ); + assert_eq!(report.job.deleted_metadata_file_count, 0); + assert!(backend.object_exists(bucket, &old).await.unwrap()); +} + +#[tokio::test] +async fn maintenance_worker_run_claims_table_entry_write_lock() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let table_path = store.paths.table_entry_path(bucket, &namespace, &table); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + let before_worker_run = backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &table_path).await; + + let report = store + .run_table_metadata_maintenance_worker_once("analytics", "sales", "orders", "worker-a".to_string()) + .await + .expect("worker run should complete"); + + assert_eq!(report.job.worker_id.as_deref(), Some("worker-a")); + assert_eq!( + backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &table_path).await, + before_worker_run + 2 + ); +} + +#[tokio::test] +async fn maintenance_scheduler_report_marks_disabled_default() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let report = store + .get_table_maintenance_scheduler_report_at(bucket, "sales", "orders", OffsetDateTime::UNIX_EPOCH + Duration::seconds(100)) + .await + .expect("scheduler report should load"); + + assert_eq!(report.status, TableMaintenanceSchedulerStatus::Disabled); + assert_eq!(report.config_source, TableMaintenanceConfigSource::Default); + assert!(!report.background_enabled); + assert_eq!( + report.recommended_actions, + vec![TableMaintenanceRecommendedAction::EnableBackgroundMaintenance] + ); + assert!(report.current_job.is_none()); + assert!(report.audit_timeline.is_empty()); + assert!(!report.quarantine.active); +} + +#[tokio::test] +async fn maintenance_scheduler_run_queues_one_durable_job() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + retain_recent_metadata_files: 2, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + + let result = store + .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) + .await + .expect("scheduler tick should queue maintenance"); + + assert_eq!(result.report.job.status, TableMetadataMaintenanceJobStatus::Queued); + assert_eq!(result.report.job.scheduler_id.as_deref(), Some("scheduler-a")); + assert!(!result.report.job.scheduler_lease_id.is_empty()); + assert_eq!(result.report.job.retain_recent_metadata_files, 2); + assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::Queued); + assert_eq!( + result.scheduler.current_job.as_ref().map(|job| job.job_id.as_str()), + Some(result.report.job.job_id.as_str()) + ); + assert_eq!( + result.report.audit_events.last().map(|event| event.action.clone()), + Some(TableMaintenanceAuditAction::SchedulerQueued) + ); +} + +#[tokio::test] +async fn maintenance_scheduler_run_persists_disabled_control_report() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let result = store + .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) + .await + .expect("disabled scheduler tick should persist a control report"); + + assert_eq!(result.report.job.status, TableMetadataMaintenanceJobStatus::Disabled); + assert_eq!(result.report.job.scheduler_id.as_deref(), Some("scheduler-a")); + let stored = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", &result.report.job.job_id) + .await + .expect("scheduler control report lookup should succeed") + .expect("scheduler control report should be durable"); + assert_eq!(stored.job.job_id, result.report.job.job_id); + assert_eq!(stored.job.status, TableMetadataMaintenanceJobStatus::Disabled); + assert_eq!( + stored.audit_events.last().map(|event| event.action.clone()), + Some(TableMaintenanceAuditAction::SchedulerControl) + ); + let scheduler = store + .get_table_maintenance_scheduler_report_at(bucket, "sales", "orders", now) + .await + .expect("scheduler report should load"); + assert_eq!(scheduler.audit_timeline.len(), 1); + assert_eq!(scheduler.audit_timeline[0].job_id, result.report.job.job_id); +} + +#[tokio::test] +async fn maintenance_scheduler_run_reuses_active_queued_job() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + worker_lease_timeout_seconds: 300, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + + let first = store + .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) + .await + .expect("first scheduler tick should queue maintenance"); + let second = store + .run_table_maintenance_scheduler_once_at( + bucket, + "sales", + "orders", + "scheduler-b".to_string(), + now + Duration::seconds(30), + ) + .await + .expect("second scheduler tick should reuse the queued job"); + + assert_eq!(second.report.job.job_id, first.report.job.job_id); + assert_eq!(second.report.job.scheduler_id.as_deref(), Some("scheduler-a")); + assert_eq!(second.report.job.status, TableMetadataMaintenanceJobStatus::Queued); + assert_eq!(second.scheduler.audit_timeline.len(), 1); +} + +#[tokio::test] +async fn maintenance_worker_claims_queued_job_before_running_it() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + let queued = store + .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) + .await + .expect("scheduler tick should queue maintenance"); + + let finished = store + .run_table_metadata_maintenance_worker_once_at( + bucket, + "sales", + "orders", + "worker-a".to_string(), + now + Duration::seconds(10), + ) + .await + .expect("worker tick should claim and finish the queued job"); + + assert_eq!(finished.job.job_id, queued.report.job.job_id); + assert_eq!(finished.job.status, TableMetadataMaintenanceJobStatus::Successful); + assert_eq!(finished.job.worker_id.as_deref(), Some("worker-a")); + assert_eq!(finished.job.attempt, 1); + assert_eq!(finished.job.scheduler_id.as_deref(), Some("scheduler-a")); + let actions = finished + .audit_events + .iter() + .map(|event| event.action.clone()) + .collect::>(); + assert_eq!( + actions, + vec![ + TableMaintenanceAuditAction::Planned, + TableMaintenanceAuditAction::SchedulerQueued, + TableMaintenanceAuditAction::WorkerStarted, + TableMaintenanceAuditAction::WorkerSucceeded, + ] + ); +} + +#[tokio::test] +async fn maintenance_worker_preserves_queued_dry_run_after_delete_is_enabled() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let table_root = format!("{}{}/", default_table_root_prefix(&namespace), table.as_str()); + let manifest_list = format!("{metadata_dir}/snap-10.avro"); + let manifest = format!("{metadata_dir}/manifest-10.avro"); + let data_file = format!("{table_root}data/part-00001.parquet"); + let orphan_data = format!("{table_root}data/orphan.parquet"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .await; + backend.seed_object(bucket, &data_file, b"data".to_vec()).await; + backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "manifest-list": manifest_list + } + ], + "refs": { + "main": { + "snapshot-id": 10, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + delete_enabled: false, + ..Default::default() + }, + ) + .await + .expect("dry-run background maintenance config should persist"); + let queued = store + .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) + .await + .expect("scheduler tick should queue dry-run maintenance"); + assert_eq!(queued.report.job.operation, TableMetadataMaintenanceOperation::DryRun); + assert_eq!(queued.report.job.deletable_object_count, 1); + + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + delete_enabled: true, + ..Default::default() + }, + ) + .await + .expect("delete-enabled background maintenance config should persist"); + let finished = store + .run_table_metadata_maintenance_worker_once_at( + bucket, + "sales", + "orders", + "worker-a".to_string(), + now + Duration::seconds(10), + ) + .await + .expect("worker tick should preserve the queued dry-run operation"); + + assert_eq!(finished.job.job_id, queued.report.job.job_id); + assert_eq!(finished.job.operation, TableMetadataMaintenanceOperation::DryRun); + assert_eq!(finished.job.status, TableMetadataMaintenanceJobStatus::Successful); + assert_eq!(finished.job.deleted_object_count, 0); + assert!(backend.object_exists(bucket, &orphan_data).await.unwrap()); +} + +#[tokio::test] +async fn maintenance_scheduler_run_recovers_expired_queued_job() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1000); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + worker_lease_timeout_seconds: 60, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + let first = store + .run_table_maintenance_scheduler_once_at(bucket, "sales", "orders", "scheduler-a".to_string(), now) + .await + .expect("first scheduler tick should queue maintenance"); + + let second = store + .run_table_maintenance_scheduler_once_at( + bucket, + "sales", + "orders", + "scheduler-b".to_string(), + now + Duration::seconds(120), + ) + .await + .expect("scheduler tick should recover expired queued maintenance"); + + assert_ne!(second.report.job.job_id, first.report.job.job_id); + let expired = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", &first.report.job.job_id) + .await + .expect("expired queued job lookup should succeed") + .expect("expired queued job should remain addressable"); + assert_eq!(expired.job.status, TableMetadataMaintenanceJobStatus::Failed); + assert!( + expired + .job + .failure_reason + .as_deref() + .is_some_and(|reason| reason.contains("scheduler lease expired")) + ); + assert_eq!( + expired.audit_events.last().map(|event| event.action.clone()), + Some(TableMaintenanceAuditAction::SchedulerLeaseExpired) + ); + assert_eq!(second.report.job.status, TableMetadataMaintenanceJobStatus::Queued); + assert_eq!(second.report.job.scheduler_id.as_deref(), Some("scheduler-b")); +} + +#[tokio::test] +async fn maintenance_scheduler_report_surfaces_active_backpressure_and_audit_timeline() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + worker_lease_timeout_seconds: 300, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + let mut running = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + running.job.status = TableMetadataMaintenanceJobStatus::Running; + running.job.worker_id = Some("worker-a".to_string()); + running.job.lease_id = "lease-a".to_string(); + running.job.heartbeat_at = Some(maintenance_timestamp(now - Duration::seconds(10))); + store + .put_table_metadata_maintenance_report(&running) + .await + .expect("running maintenance report should be seeded"); + + let report = store + .get_table_maintenance_scheduler_report_at(bucket, "sales", "orders", now) + .await + .expect("scheduler report should load"); + + assert_eq!(report.status, TableMaintenanceSchedulerStatus::Backpressured); + assert_eq!( + report.current_job.as_ref().map(|job| job.job_id.as_str()), + Some(running.job.job_id.as_str()) + ); + assert_eq!(report.recommended_actions, vec![TableMaintenanceRecommendedAction::WaitForActiveWorker]); + assert_eq!(report.audit_timeline.len(), 1); + assert_eq!(report.audit_timeline[0].job_id, running.job.job_id); + assert_eq!(report.audit_timeline[0].status, TableMetadataMaintenanceJobStatus::Running); + assert_eq!(report.audit_timeline[0].worker_id.as_deref(), Some("worker-a")); +} + +#[tokio::test] +async fn maintenance_scheduler_report_surfaces_quarantine_boundary() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + quarantine_enabled: true, + quarantine_retention_seconds: 86_400, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + let mut failed = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + failed.job.status = TableMetadataMaintenanceJobStatus::Failed; + failed.job.failure_reason = Some("quarantine retained failed cleanup candidates".to_string()); + failed.job.quarantine_enabled = true; + failed.job.quarantine_retention_seconds = 86_400; + failed.job.quarantined_object_count = 2; + failed.job.finished_at = Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH + Duration::seconds(90))); + store + .put_table_metadata_maintenance_report(&failed) + .await + .expect("failed maintenance report should be seeded"); + + let report = store + .get_table_maintenance_scheduler_report_at(bucket, "sales", "orders", OffsetDateTime::UNIX_EPOCH + Duration::seconds(100)) + .await + .expect("scheduler report should load"); + + assert_eq!(report.status, TableMaintenanceSchedulerStatus::Quarantined); + assert!(report.quarantine.active); + assert_eq!(report.quarantine.retention_seconds, 86_400); + assert_eq!(report.quarantine.quarantined_object_count, 2); + assert_eq!(report.quarantine.source_job_id.as_deref(), Some(failed.job.job_id.as_str())); + assert!( + report + .recommended_actions + .contains(&TableMaintenanceRecommendedAction::ReviewQuarantine) + ); +} + +#[tokio::test] +async fn maintenance_quarantine_retry_clears_boundary_and_unblocks_scheduler() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let now = OffsetDateTime::now_utc(); + let (_namespace, _table, failed) = + seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; + + let result = store + .apply_table_maintenance_quarantine_operation( + bucket, + "sales", + "orders", + &failed.job.job_id, + TableMaintenanceQuarantineOperationRequest { + action: TableMaintenanceQuarantineAction::Retry, + reason: Some("operator reviewed retained candidates".to_string()), + }, + ) + .await + .expect("quarantine retry should update the current maintenance job"); + + assert_eq!(result.action, TableMaintenanceQuarantineAction::Retry); + assert_eq!(result.report.job.job_id, failed.job.job_id); + assert_eq!(result.report.job.quarantined_object_count, 0); + assert!(result.report.job.next_retry_after.is_none()); + let event = result + .report + .audit_events + .last() + .expect("quarantine retry should append an audit event"); + assert_eq!(event.action, TableMaintenanceAuditAction::QuarantineRetry); + assert_eq!(event.actor, TableMaintenanceAuditActor::Operator); + assert_eq!(event.reason.as_deref(), Some("operator reviewed retained candidates")); + assert_eq!(event.before_status, Some(TableMetadataMaintenanceJobStatus::Failed)); + assert_eq!(event.after_status, Some(TableMetadataMaintenanceJobStatus::Failed)); + assert_eq!(event.before_quarantined_object_count, Some(2)); + assert_eq!(event.after_quarantined_object_count, Some(0)); + assert!( + !result + .report + .job + .recommended_actions + .contains(&TableMaintenanceRecommendedAction::ReviewQuarantine) + ); + assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::Ready); + assert!(!result.scheduler.quarantine.active); + let summary = result + .scheduler + .audit_timeline + .iter() + .find(|summary| summary.job_id == failed.job.job_id) + .expect("scheduler timeline should include the retried job"); + assert_eq!( + summary.audit_events.last().map(|event| event.action.clone()), + Some(TableMaintenanceAuditAction::QuarantineRetry) + ); + + let current_report = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", MAINTENANCE_JOB_ALIAS_CURRENT) + .await + .expect("current maintenance report should load") + .expect("current maintenance report should exist"); + assert_eq!(current_report.job.job_id, failed.job.job_id); + assert_eq!(current_report.job.quarantined_object_count, 0); +} + +#[tokio::test] +async fn maintenance_quarantine_inspect_reports_without_mutating_current_job() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let now = OffsetDateTime::now_utc(); + let (_namespace, _table, failed) = + seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; + + let result = store + .apply_table_maintenance_quarantine_operation( + bucket, + "sales", + "orders", + &failed.job.job_id, + TableMaintenanceQuarantineOperationRequest { + action: TableMaintenanceQuarantineAction::Inspect, + reason: Some("ignored for inspect".to_string()), + }, + ) + .await + .expect("quarantine inspect should load the maintenance job"); + + assert_eq!(result.action, TableMaintenanceQuarantineAction::Inspect); + assert_eq!(result.report.job.job_id, failed.job.job_id); + assert_eq!(result.report.job.quarantined_object_count, 2); + let expected_retry_after = maintenance_timestamp(now + Duration::seconds(300)); + assert_eq!(result.report.job.next_retry_after.as_deref(), Some(expected_retry_after.as_str())); + assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::RetryDeferred); + + let current_report = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", MAINTENANCE_JOB_ALIAS_CURRENT) + .await + .expect("current maintenance report should load") + .expect("current maintenance report should exist"); + assert_eq!(current_report.job.quarantined_object_count, 2); + assert_eq!(current_report.audit_events, failed.audit_events); +} + +#[tokio::test] +async fn maintenance_quarantine_release_preserves_retry_deferral() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let now = OffsetDateTime::now_utc(); + let (_namespace, _table, failed) = + seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; + + let result = store + .apply_table_maintenance_quarantine_operation( + bucket, + "sales", + "orders", + &failed.job.job_id, + TableMaintenanceQuarantineOperationRequest { + action: TableMaintenanceQuarantineAction::Release, + reason: Some("objects retained for later retry".to_string()), + }, + ) + .await + .expect("quarantine release should update the current maintenance job"); + + assert_eq!(result.action, TableMaintenanceQuarantineAction::Release); + assert_eq!(result.report.job.quarantined_object_count, 0); + let expected_retry_after = maintenance_timestamp(now + Duration::seconds(300)); + assert_eq!(result.report.job.next_retry_after.as_deref(), Some(expected_retry_after.as_str())); + assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::RetryDeferred); + assert!(result.report.job.failure_reason.as_deref().is_some_and(|reason| { + reason.contains("maintenance quarantine released by operator") && reason.contains("objects retained for later retry") + })); +} + +#[tokio::test] +async fn maintenance_quarantine_abandon_clears_boundary_and_retry() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let now = OffsetDateTime::now_utc(); + let (_namespace, _table, failed) = + seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; + + let result = store + .apply_table_maintenance_quarantine_operation( + bucket, + "sales", + "orders", + &failed.job.job_id, + TableMaintenanceQuarantineOperationRequest { + action: TableMaintenanceQuarantineAction::Abandon, + reason: Some("operator accepted retained objects".to_string()), + }, + ) + .await + .expect("quarantine abandon should update the current maintenance job"); + + assert_eq!(result.action, TableMaintenanceQuarantineAction::Abandon); + assert_eq!(result.report.job.quarantined_object_count, 0); + assert!(result.report.job.next_retry_after.is_none()); + assert_eq!(result.scheduler.status, TableMaintenanceSchedulerStatus::Ready); + assert!(result.report.job.failure_reason.as_deref().is_some_and(|reason| { + reason.contains("maintenance quarantine abandoned by operator") && reason.contains("operator accepted retained objects") + })); +} + +#[tokio::test] +async fn maintenance_quarantine_rejects_mutating_non_current_job() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let now = OffsetDateTime::now_utc(); + let (_namespace, _table, old_failed) = + seed_quarantined_table_maintenance(&store, &backend, bucket, now, Some(now + Duration::seconds(300))).await; + let mut current_failed = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + current_failed.job.status = TableMetadataMaintenanceJobStatus::Failed; + current_failed.job.quarantine_enabled = true; + current_failed.job.quarantine_retention_seconds = 86_400; + current_failed.job.quarantined_object_count = 1; + store + .put_table_metadata_maintenance_report(¤t_failed) + .await + .expect("new current maintenance report should be seeded"); + + let error = store + .apply_table_maintenance_quarantine_operation( + bucket, + "sales", + "orders", + &old_failed.job.job_id, + TableMaintenanceQuarantineOperationRequest { + action: TableMaintenanceQuarantineAction::Release, + reason: None, + }, + ) + .await + .expect_err("mutating a non-current quarantine job should fail"); + + assert_eq!(error, TableCatalogStoreError::Conflict("maintenance job is not current".to_string())); +} + +#[tokio::test] +async fn maintenance_worker_run_records_audit_timeline_events() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + background_enabled: true, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + + let report = store + .run_table_metadata_maintenance_worker_once(bucket, "sales", "orders", "worker-a".to_string()) + .await + .expect("maintenance worker should finish"); + + let actions = report + .audit_events + .iter() + .map(|event| event.action.clone()) + .collect::>(); + assert_eq!( + actions, + vec![ + TableMaintenanceAuditAction::Planned, + TableMaintenanceAuditAction::WorkerStarted, + TableMaintenanceAuditAction::WorkerSucceeded, + ] + ); + assert_eq!(report.audit_events[1].actor, TableMaintenanceAuditActor::Worker); + assert_eq!(report.audit_events[1].before_status, Some(TableMetadataMaintenanceJobStatus::Successful)); + assert_eq!(report.audit_events[2].after_status, Some(TableMetadataMaintenanceJobStatus::Successful)); + + let scheduler = store + .get_table_maintenance_scheduler_report(bucket, "sales", "orders") + .await + .expect("scheduler report should load"); + let summary = scheduler.current_job.expect("current job should be visible"); + assert_eq!(summary.job_id, report.job.job_id); + assert_eq!(summary.audit_events, report.audit_events); +} + +#[tokio::test] +async fn maintenance_heartbeat_appends_worker_audit_event() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + let mut running = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + running.job.status = TableMetadataMaintenanceJobStatus::Running; + running.job.worker_id = Some("worker-a".to_string()); + running.job.lease_id = "lease-a".to_string(); + running.job.heartbeat_at = Some(maintenance_timestamp(now - Duration::seconds(10))); + store + .put_table_metadata_maintenance_report(&running) + .await + .expect("running maintenance report should be seeded"); + + let heartbeat = store + .heartbeat_table_metadata_maintenance_job_at( + TableMaintenanceHeartbeatRef { + table_bucket: bucket, + namespace: "sales", + table: "orders", + job_id: &running.job.job_id, + lease_id: "lease-a", + worker_id: "worker-a", + }, + now, + ) + .await + .expect("heartbeat should update the running job"); + + let event = heartbeat.audit_events.last().expect("heartbeat should append an audit event"); + assert_eq!(event.action, TableMaintenanceAuditAction::WorkerHeartbeat); + assert_eq!(event.actor, TableMaintenanceAuditActor::Worker); + assert_eq!(event.before_status, Some(TableMetadataMaintenanceJobStatus::Running)); + assert_eq!(event.after_status, Some(TableMetadataMaintenanceJobStatus::Running)); +} + +#[tokio::test] +async fn maintenance_worker_run_defers_until_retry_after() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 0, + delete_enabled: false, + background_enabled: true, + max_retry_attempts: 2, + retry_initial_backoff_seconds: 60, + retry_max_backoff_seconds: 60, + ..Default::default() + }, + ) + .await + .expect("retry-enabled maintenance config should persist"); + let mut failed = store + .run_table_metadata_maintenance(bucket, "sales", "orders", true, Some("worker-a".to_string())) + .await + .expect("delete failure should be recorded when delete is disabled"); + failed.job.next_retry_after = Some(maintenance_timestamp(now + Duration::seconds(30))); + store + .put_table_metadata_maintenance_report(&failed) + .await + .expect("failed retry report should be seeded"); + + let deferred = store + .run_table_metadata_maintenance_worker_once_at(bucket, "sales", "orders", "worker-b".to_string(), now) + .await + .expect("worker tick should defer while retry backoff is active"); + + assert_eq!(deferred.job.job_id, failed.job.job_id); + assert_eq!(deferred.job.status, TableMetadataMaintenanceJobStatus::Failed); + assert_eq!(deferred.job.worker_id.as_deref(), Some("worker-a")); + assert!( + deferred + .job + .recommended_actions + .contains(&TableMaintenanceRecommendedAction::WaitForRetryBackoff) + ); + assert!(backend.object_exists(bucket, &old).await.unwrap()); +} + +#[tokio::test] +async fn maintenance_worker_run_backpressures_active_running_job() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 0, + delete_enabled: false, + background_enabled: true, + worker_lease_timeout_seconds: 300, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + let mut running = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + running.job.status = TableMetadataMaintenanceJobStatus::Running; + running.job.worker_id = Some("worker-a".to_string()); + running.job.lease_id = "lease-a".to_string(); + running.job.heartbeat_at = Some(maintenance_timestamp(now - Duration::seconds(10))); + store + .put_table_metadata_maintenance_report(&running) + .await + .expect("running maintenance report should be seeded"); + + let report = store + .run_table_metadata_maintenance_worker_once_at(bucket, "sales", "orders", "worker-b".to_string(), now) + .await + .expect("worker tick should return the active running job"); + + assert_eq!(report.job.job_id, running.job.job_id); + assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Running); + assert_eq!(report.job.worker_id.as_deref(), Some("worker-a")); + assert_eq!( + report.job.recommended_actions, + vec![TableMaintenanceRecommendedAction::WaitForActiveWorker] + ); +} + +#[tokio::test] +async fn maintenance_worker_run_recovers_expired_running_job() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1000); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + store + .put_table_maintenance_config( + bucket, + "sales", + "orders", + TableMaintenanceConfig { + version: TABLE_MAINTENANCE_CONFIG_VERSION, + retain_recent_metadata_files: 0, + delete_enabled: false, + background_enabled: true, + worker_lease_timeout_seconds: 60, + ..Default::default() + }, + ) + .await + .expect("background maintenance config should persist"); + let mut running = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + let expired_job_id = running.job.job_id.clone(); + running.job.status = TableMetadataMaintenanceJobStatus::Running; + running.job.worker_id = Some("worker-a".to_string()); + running.job.lease_id = "lease-a".to_string(); + running.job.heartbeat_at = Some(maintenance_timestamp(now - Duration::seconds(120))); + store + .put_table_metadata_maintenance_report(&running) + .await + .expect("expired running maintenance report should be seeded"); + + let report = store + .run_table_metadata_maintenance_worker_once_at(bucket, "sales", "orders", "worker-b".to_string(), now) + .await + .expect("worker tick should recover expired running job and run again"); + + assert_ne!(report.job.job_id, expired_job_id); + assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Successful); + assert_eq!(report.job.worker_id.as_deref(), Some("worker-b")); + + let expired = store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", &expired_job_id) + .await + .expect("expired job lookup should succeed") + .expect("expired job should remain addressable"); + assert_eq!(expired.job.status, TableMetadataMaintenanceJobStatus::Failed); + assert!( + expired + .job + .failure_reason + .as_deref() + .is_some_and(|reason| reason.contains("lease expired")) + ); + assert_eq!( + expired.job.recommended_actions, + vec![TableMaintenanceRecommendedAction::InvestigateFailure] + ); + let event = expired + .audit_events + .last() + .expect("expired lease recovery should append an audit event"); + assert_eq!(event.action, TableMaintenanceAuditAction::WorkerLeaseExpired); + assert_eq!(event.actor, TableMaintenanceAuditActor::Scheduler); + assert_eq!(event.before_status, Some(TableMetadataMaintenanceJobStatus::Running)); + assert_eq!(event.after_status, Some(TableMetadataMaintenanceJobStatus::Failed)); +} + +#[tokio::test] +async fn maintenance_worker_heartbeat_updates_current_running_job() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let first = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + let second = OffsetDateTime::UNIX_EPOCH + Duration::seconds(130); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend + .seed_object( + bucket, + &default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"), + br#"{"metadata-log":[]}"#.to_vec(), + ) + .await; + let mut running = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + running.job.status = TableMetadataMaintenanceJobStatus::Running; + running.job.worker_id = Some("worker-a".to_string()); + running.job.lease_id = "lease-a".to_string(); + running.job.heartbeat_at = Some(maintenance_timestamp(first)); + let job_id = running.job.job_id.clone(); + store + .put_table_metadata_maintenance_report(&running) + .await + .expect("running maintenance report should be seeded"); + + let heartbeat = store + .heartbeat_table_metadata_maintenance_job_at( + TableMaintenanceHeartbeatRef { + table_bucket: bucket, + namespace: "sales", + table: "orders", + job_id: &job_id, + lease_id: "lease-a", + worker_id: "worker-a", + }, + second, + ) + .await + .expect("heartbeat should update the current running job"); + + assert_eq!(heartbeat.job.job_id, job_id); + assert_eq!(heartbeat.job.status, TableMetadataMaintenanceJobStatus::Running); + assert_eq!(heartbeat.job.heartbeat_at.as_deref(), Some(maintenance_timestamp(second).as_str())); +} + +#[tokio::test] +async fn maintenance_reachability_reports_manifest_lists_as_manual_review() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let manifest_list = format!("{}/snap-10.avro", default_table_metadata_dir_path(&namespace, &table)); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend.seed_object(bucket, &manifest_list, b"avro".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [], + "schemas": [], + "partition-specs": [], + "sort-orders": [], + "snapshots": [ + { + "snapshot-id": 10, + "manifest-list": manifest_list + } + ], + "snapshot-log": [ + { + "timestamp-ms": 1, + "snapshot-id": 10 + } + ], + "refs": { + "main": { + "snapshot-id": 10, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("metadata maintenance dry-run should succeed"); + + assert_eq!(report.cleanup_candidate_locations, vec![old]); + let manifest_report = report + .referenced_object_reports + .iter() + .find(|object| object.object_location == manifest_list) + .expect("manifest list should be reported as a referenced object"); + assert_eq!(manifest_report.object_kind, TableMetadataMaintenanceObjectKind::ManifestList); + assert_eq!(manifest_report.state, TableMetadataMaintenanceObjectState::ManualReviewRequired); + assert_eq!( + manifest_report.reasons, + vec![ + TableMetadataMaintenanceReason::ManifestList, + TableMetadataMaintenanceReason::UnsupportedManifestAvro, + ] + ); + assert_eq!( + report.reachability_graph.status, + TableMaintenanceReachabilityGraphStatus::ManualReviewRequired + ); + assert_eq!(report.reachability_graph.metadata_file_count, 2); + assert_eq!(report.reachability_graph.manifest_list_count, 1); + assert_eq!(report.reachability_graph.manual_review_count, 1); + assert!( + report + .reachability_graph + .reasons + .contains(&TableMaintenanceReachabilityGraphReason::ManifestAvroReaderUnavailable) + ); +} + +#[tokio::test] +async fn maintenance_reachability_expands_manifest_avro_references() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let table_root = format!("{}{}/", default_table_root_prefix(&namespace), table.as_str()); + let manifest_list = format!("{metadata_dir}/snap-10.avro"); + let manifest = format!("{metadata_dir}/manifest-10.avro"); + let data_file = format!("{table_root}data/part-00001.parquet"); + let delete_file = format!("{table_root}delete/pos-00001.parquet"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0), (&delete_file, 1)])) + .await; + backend.seed_object(bucket, &data_file, b"data".to_vec()).await; + backend.seed_object(bucket, &delete_file, b"delete".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "manifest-list": manifest_list + } + ], + "refs": { + "main": { + "snapshot-id": 10, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("metadata maintenance dry-run should succeed"); + + assert_eq!(report.reachability_graph.status, TableMaintenanceReachabilityGraphStatus::Complete); + assert_eq!(report.reachability_graph.manifest_list_count, 1); + assert_eq!(report.reachability_graph.manifest_file_count, 1); + assert_eq!(report.reachability_graph.data_file_count, 1); + assert_eq!(report.reachability_graph.delete_file_count, 1); + assert_eq!(report.reachability_graph.manual_review_count, 0); + assert!( + !report + .reachability_graph + .reasons + .contains(&TableMaintenanceReachabilityGraphReason::ManifestAvroReaderUnavailable) + ); + for (location, kind) in [ + (&manifest_list, TableMetadataMaintenanceObjectKind::ManifestList), + (&manifest, TableMetadataMaintenanceObjectKind::ManifestFile), + (&data_file, TableMetadataMaintenanceObjectKind::DataFile), + (&delete_file, TableMetadataMaintenanceObjectKind::DeleteFile), + ] { + let referenced = report + .referenced_object_reports + .iter() + .find(|object| object.object_location == *location) + .expect("referenced object should be reported"); + assert_eq!(referenced.object_kind, kind); + assert_eq!(referenced.state, TableMetadataMaintenanceObjectState::Retained); + } + assert!(report.cleanup_object_candidate_locations.is_empty()); + assert!(report.deletable_object_locations.is_empty()); +} + +#[tokio::test] +async fn maintenance_reachability_treats_v1_snapshot_manifests_as_reachable() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let data_dir = default_table_data_dir_path(&namespace, &table); + let manifest = format!("{metadata_dir}/manifest-10.avro"); + let data_file = format!("{data_dir}/part-00001.parquet"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .await; + backend.seed_object(bucket, &data_file, b"data".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "manifests": [manifest] + } + ], + "refs": { + "main": { + "snapshot-id": 10, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("metadata maintenance dry-run should succeed"); + + assert_eq!(report.reachability_graph.status, TableMaintenanceReachabilityGraphStatus::Complete); + assert_eq!(report.reachability_graph.manifest_file_count, 1); + assert_eq!(report.reachability_graph.data_file_count, 1); + assert!(report.cleanup_object_candidate_locations.is_empty()); + assert!(report.deletable_object_locations.is_empty()); + for location in [&manifest, &data_file] { + let referenced = report + .referenced_object_reports + .iter() + .find(|object| object.object_location == *location) + .expect("v1 manifest reference should be retained"); + assert_eq!(referenced.state, TableMetadataMaintenanceObjectState::Retained); + } +} + +#[tokio::test] +async fn maintenance_reachability_uses_table_warehouse_object_paths() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let manifest_list = "tables/table-id/metadata/snap-10.avro".to_string(); + let manifest = "tables/table-id/metadata/manifest-10.avro".to_string(); + let data_file = "tables/table-id/data/part-00001.parquet".to_string(); + let orphan_data = "tables/table-id/data/orphan.parquet".to_string(); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .await; + backend.seed_object(bucket, &data_file, b"data".to_vec()).await; + backend.seed_object(bucket, &orphan_data, b"orphan".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "manifest-list": format!("s3://{bucket}/{manifest_list}") + } + ], + "refs": { + "main": { + "snapshot-id": 10, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("metadata maintenance dry-run should succeed"); + + assert_eq!(report.reachability_graph.status, TableMaintenanceReachabilityGraphStatus::Complete); + assert!( + report.referenced_object_reports.iter().any( + |object| object.object_location == manifest_list && object.state == TableMetadataMaintenanceObjectState::Retained + ) + ); + assert!( + report + .referenced_object_reports + .iter() + .any(|object| object.object_location == data_file && object.state == TableMetadataMaintenanceObjectState::Retained) + ); + assert_eq!(report.cleanup_object_candidate_locations, vec![orphan_data.clone()]); + assert_eq!(report.deletable_object_locations, vec![orphan_data]); +} + +#[tokio::test] +async fn maintenance_reachability_fails_closed_when_retained_metadata_is_unreadable() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let orphan_data = format!("{}/orphan.parquet", default_table_data_dir_path(&namespace, &table)); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"not-json".to_vec()).await; + backend.seed_object(bucket, &orphan_data, b"orphan".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [ + { + "timestamp-ms": 1, + "metadata-file": old + } + ], + "snapshots": [] + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("metadata maintenance dry-run should succeed"); + + assert_eq!( + report.reachability_graph.status, + TableMaintenanceReachabilityGraphStatus::ManualReviewRequired + ); + assert!(report.cleanup_object_candidate_locations.is_empty()); + assert!(report.deletable_object_locations.is_empty()); + let retained_metadata = report + .referenced_object_reports + .iter() + .find(|object| object.object_location == old) + .expect("unreadable retained metadata should be reported"); + assert_eq!(retained_metadata.object_kind, TableMetadataMaintenanceObjectKind::MetadataFile); + assert_eq!(retained_metadata.state, TableMetadataMaintenanceObjectState::ManualReviewRequired); + assert!( + retained_metadata + .reasons + .contains(&TableMetadataMaintenanceReason::UnreadableMetadata) + ); +} + +#[tokio::test] +async fn maintenance_dry_run_reports_unreachable_manifest_data_and_delete_candidates() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let table_root = format!("{}{}/", default_table_root_prefix(&namespace), table.as_str()); + let manifest_list = format!("{metadata_dir}/snap-10.avro"); + let manifest = format!("{metadata_dir}/manifest-10.avro"); + let data_file = format!("{table_root}data/part-00001.parquet"); + let orphan_manifest = format!("{metadata_dir}/manifest-orphan.avro"); + let orphan_data = format!("{table_root}data/orphan.parquet"); + let orphan_delete = format!("{table_root}delete/orphan-delete.parquet"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .await; + backend.seed_object(bucket, &data_file, b"data".to_vec()).await; + backend.seed_object(bucket, &orphan_manifest, manifest_avro_bytes(&[])).await; + backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; + backend.seed_object(bucket, &orphan_delete, b"orphan-delete".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "manifest-list": manifest_list + } + ], + "refs": { + "main": { + "snapshot-id": 10, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("metadata maintenance dry-run should succeed"); + let candidates = report + .cleanup_object_candidate_locations + .iter() + .cloned() + .collect::>(); + let deletable = report.deletable_object_locations.iter().cloned().collect::>(); + let expected = BTreeSet::from([orphan_data.clone(), orphan_delete.clone(), orphan_manifest.clone()]); + + assert_eq!(candidates, expected); + assert_eq!(deletable, expected); + assert_eq!( + object_cleanup_report(&report, &orphan_manifest).object_kind, + TableMetadataMaintenanceObjectKind::ManifestFile + ); + assert_eq!( + object_cleanup_report(&report, &orphan_data).object_kind, + TableMetadataMaintenanceObjectKind::DataFile + ); + assert_eq!( + object_cleanup_report(&report, &orphan_delete).object_kind, + TableMetadataMaintenanceObjectKind::DeleteFile + ); +} + +#[tokio::test] +async fn maintenance_delete_removes_only_planned_unreachable_table_objects() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let table_root = format!("{}{}/", default_table_root_prefix(&namespace), table.as_str()); + let manifest_list = format!("{metadata_dir}/snap-10.avro"); + let manifest = format!("{metadata_dir}/manifest-10.avro"); + let data_file = format!("{table_root}data/part-00001.parquet"); + let orphan_manifest = format!("{metadata_dir}/manifest-orphan.avro"); + let orphan_data = format!("{table_root}data/orphan.parquet"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .await; + backend.seed_object(bucket, &data_file, b"data".to_vec()).await; + backend.seed_object(bucket, &orphan_manifest, manifest_avro_bytes(&[])).await; + backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "manifest-list": manifest_list + } + ], + "refs": { + "main": { + "snapshot-id": 10, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let deleted = store + .delete_table_metadata_maintenance_candidates(bucket, "sales", "orders", 0) + .await + .expect("maintenance delete should succeed"); + + assert_eq!( + deleted.deletable_object_locations.iter().cloned().collect::>(), + BTreeSet::from([orphan_data.clone(), orphan_manifest.clone()]) + ); + assert!(backend.object_exists(bucket, &manifest_list).await.unwrap()); + assert!(backend.object_exists(bucket, &manifest).await.unwrap()); + assert!(backend.object_exists(bucket, &data_file).await.unwrap()); + assert!(!backend.object_exists(bucket, &orphan_manifest).await.unwrap()); + assert!(!backend.object_exists(bucket, &orphan_data).await.unwrap()); +} + +#[tokio::test] +async fn maintenance_dry_run_keeps_metadata_log_references() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let v1 = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let logged = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let v3 = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + for metadata in [&v1, &logged, &v3] { + backend.seed_object(bucket, metadata, b"{}".to_vec()).await; + } + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [ + { + "timestamp-ms": 1, + "metadata-file": logged + } + ] + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + + assert!(report.retained_metadata_locations.contains(¤t)); + assert!(report.retained_metadata_locations.contains(&logged)); + assert_eq!(report.cleanup_candidate_locations, vec![v1, v3]); +} + +#[tokio::test] +async fn maintenance_dry_run_keeps_metadata_for_protected_snapshot_refs() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let orphan = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let tagged = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let unreferenced = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &orphan, b"{}".to_vec()).await; + backend + .seed_object( + bucket, + &tagged, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 10 + })) + .unwrap(), + ) + .await; + backend + .seed_object( + bucket, + &unreferenced, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 20 + })) + .unwrap(), + ) + .await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 30, + "metadata-log": [], + "refs": { + "main": { + "snapshot-id": 30, + "type": "branch" + }, + "audit": { + "snapshot-id": 10, + "type": "tag" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + + assert!(report.retained_metadata_locations.contains(&tagged)); + assert_eq!(report.cleanup_candidate_locations, vec![orphan, unreferenced]); +} + +#[tokio::test] +async fn snapshot_expiration_plan_retains_current_recent_and_protected_refs() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 30, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "timestamp-ms": 1000, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-10.avro" + }, + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" + }, + { + "snapshot-id": 30, + "timestamp-ms": 3000, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-30.avro" + } + ], + "refs": { + "main": { + "snapshot-id": 30, + "type": "branch" + }, + "audit": { + "snapshot-id": 10, + "type": "tag" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_snapshot_expiration( + bucket, + "sales", + "orders", + TableSnapshotExpirationConfig { + min_snapshots_to_keep: 1, + max_snapshot_age_ms: 1, + }, + ) + .await + .expect("snapshot expiration planning should succeed"); + + let current_snapshot = snapshot_expiration_report(&report, 30); + assert_eq!(current_snapshot.state, TableSnapshotExpirationSnapshotState::Retained); + assert!( + current_snapshot + .reasons + .contains(&TableSnapshotExpirationReason::CurrentSnapshot) + ); + + let protected_snapshot = snapshot_expiration_report(&report, 10); + assert_eq!(protected_snapshot.state, TableSnapshotExpirationSnapshotState::ManualReviewRequired); + assert!( + protected_snapshot + .reasons + .contains(&TableSnapshotExpirationReason::ProtectedSnapshotRef) + ); + assert!( + protected_snapshot + .reasons + .contains(&TableSnapshotExpirationReason::UserDefinedSnapshotRef) + ); + + let expired_snapshot = snapshot_expiration_report(&report, 20); + assert_eq!(expired_snapshot.state, TableSnapshotExpirationSnapshotState::ExpirationCandidate); + assert!( + expired_snapshot + .reasons + .contains(&TableSnapshotExpirationReason::SnapshotAgeExpired) + ); + assert_eq!(report.expiration_candidate_count, 1); + assert_eq!(report.manual_review_count, 1); +} + +#[tokio::test] +async fn snapshot_expiration_plan_fails_closed_for_table_retention_property_conflicts() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 20, + "metadata-log": [], + "properties": { + "history.expire.min-snapshots-to-keep": "5" + }, + "snapshots": [ + { + "snapshot-id": 10, + "timestamp-ms": 1000, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-10.avro" + }, + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" + } + ], + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_snapshot_expiration( + bucket, + "sales", + "orders", + TableSnapshotExpirationConfig { + min_snapshots_to_keep: 1, + max_snapshot_age_ms: 1, + }, + ) + .await + .expect("snapshot expiration planning should succeed"); + + assert_eq!(report.expiration_candidate_count, 0); + assert_eq!(report.manual_review_count, 2); + for snapshot in &report.snapshot_reports { + assert_eq!(snapshot.state, TableSnapshotExpirationSnapshotState::ManualReviewRequired); + assert!( + snapshot + .reasons + .contains(&TableSnapshotExpirationReason::TableRetentionPropertyConflict) + ); + } +} + +#[tokio::test] +async fn snapshot_expiration_plan_requires_snapshot_timestamps() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 20, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 10, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-10.avro" + }, + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" + } + ], + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_snapshot_expiration( + bucket, + "sales", + "orders", + TableSnapshotExpirationConfig { + min_snapshots_to_keep: 1, + max_snapshot_age_ms: 1, + }, + ) + .await + .expect("snapshot expiration planning should succeed"); + + let missing_timestamp = snapshot_expiration_report(&report, 10); + assert_eq!(missing_timestamp.state, TableSnapshotExpirationSnapshotState::ManualReviewRequired); + assert!( + missing_timestamp + .reasons + .contains(&TableSnapshotExpirationReason::MissingSnapshotTimestamp) + ); + assert_eq!(report.expiration_candidate_count, 0); + assert_eq!(report.manual_review_count, 1); +} + +#[tokio::test] +async fn compaction_plan_reports_manifest_reader_gap_without_rewrite_candidates() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 20, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" + } + ], + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_compaction( + bucket, + "sales", + "orders", + TableCompactionPlanningConfig { + target_file_size_bytes: 512 * 1024 * 1024, + small_file_threshold_bytes: 64 * 1024 * 1024, + min_input_files: 2, + max_rewrite_bytes_per_job: 1024 * 1024 * 1024, + }, + ) + .await + .expect("compaction planning should succeed"); + + assert_eq!(report.status, TableCompactionPlanningStatus::ManualReviewRequired); + assert_eq!(report.candidate_file_count, 0); + assert_eq!(report.rewrite_group_count, 0); + assert_eq!(report.manual_review_count, 1); + let snapshot = compaction_snapshot_report(&report, 20); + assert_eq!(snapshot.status, TableCompactionPlanningStatus::ManualReviewRequired); + assert!(snapshot.reasons.contains(&TableCompactionPlanningReason::ManifestList)); + assert!( + snapshot + .reasons + .contains(&TableCompactionPlanningReason::ManifestAvroReaderUnavailable) + ); +} + +#[tokio::test] +async fn compaction_plan_reports_row_level_delete_files_without_rewrite_candidates() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let data_dir = default_table_data_dir_path(&namespace, &table); + let delete_dir = default_table_delete_dir_path(&namespace, &table); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + let manifest_list = format!("{metadata_dir}/snap-20.avro"); + let manifest = format!("{metadata_dir}/manifest-20.avro"); + let data_file = format!("{data_dir}/part-left.parquet"); + let position_delete_file = format!("{delete_dir}/pos-left.parquet"); + let equality_delete_file = format!("{delete_dir}/eq-left.parquet"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object( + bucket, + &manifest, + manifest_avro_bytes(&[(&data_file, 0), (&position_delete_file, 1), (&equality_delete_file, 2)]), + ) + .await; + backend.seed_object(bucket, &data_file, parquet_i32_bytes(&[1, 2])).await; + backend + .seed_object(bucket, &position_delete_file, b"position-delete".to_vec()) + .await; + backend + .seed_object(bucket, &equality_delete_file, b"equality-delete".to_vec()) + .await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 20, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": manifest_list + } + ], + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + })) + .expect("current metadata should serialize"), + ) + .await; + + let config = TableCompactionPlanningConfig { + target_file_size_bytes: 512 * 1024 * 1024, + small_file_threshold_bytes: 64 * 1024 * 1024, + min_input_files: 2, + max_rewrite_bytes_per_job: 1024 * 1024 * 1024, + }; + let report = store + .plan_table_compaction(bucket, "sales", "orders", config.clone()) + .await + .expect("compaction planning should succeed"); + + assert_eq!(report.status, TableCompactionPlanningStatus::ManualReviewRequired); + assert_eq!(report.candidate_file_count, 1); + assert_eq!(report.rewrite_group_count, 0); + assert_eq!(report.manual_review_count, 1); + assert_eq!( + report.row_level_planning.status, + TableRowLevelMaintenancePlanningStatus::ManualReviewRequired + ); + assert_eq!(report.row_level_planning.delete_file_count, 2); + assert_eq!(report.row_level_planning.position_delete_file_count, 1); + assert_eq!(report.row_level_planning.equality_delete_file_count, 1); + assert!( + report + .row_level_planning + .reasons + .contains(&TableRowLevelMaintenancePlanningReason::DeleteFileRewriteUnsupported) + ); + assert!( + report + .row_level_planning + .delete_files + .iter() + .any(|delete_file| delete_file.file_location == position_delete_file + && delete_file.content == TableRowLevelDeleteFileContent::PositionDelete + && delete_file.object_exists) + ); + assert!( + report + .row_level_planning + .delete_files + .iter() + .any(|delete_file| delete_file.file_location == equality_delete_file + && delete_file.content == TableRowLevelDeleteFileContent::EqualityDelete + && delete_file.object_exists) + ); + let snapshot = compaction_snapshot_report(&report, 20); + assert_eq!(snapshot.status, TableCompactionPlanningStatus::ManualReviewRequired); + assert!(snapshot.reasons.contains(&TableCompactionPlanningReason::DeleteFile)); + assert!( + snapshot + .reasons + .contains(&TableCompactionPlanningReason::RowLevelRewriteUnsupported) + ); + + let err = store + .commit_table_compaction(bucket, "sales", "orders", config) + .await + .expect_err("delete-file compaction should fail closed"); + assert!( + err.to_string().contains("compaction has no safe rewrite candidates"), + "unexpected error: {err}" + ); +} + +#[tokio::test] +async fn compaction_plan_requires_current_snapshot_metadata() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 30, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 20, + "timestamp-ms": 2000, + "manifest-list": "s3://analytics/tables/table-id/metadata/snap-20.avro" + } + ] + })) + .unwrap(), + ) + .await; + + let report = store + .plan_table_compaction( + bucket, + "sales", + "orders", + TableCompactionPlanningConfig { + target_file_size_bytes: 512 * 1024 * 1024, + small_file_threshold_bytes: 64 * 1024 * 1024, + min_input_files: 2, + max_rewrite_bytes_per_job: 1024 * 1024 * 1024, + }, + ) + .await + .expect("compaction planning should succeed"); + + assert_eq!(report.status, TableCompactionPlanningStatus::ManualReviewRequired); + assert_eq!(report.manual_review_count, 1); + let snapshot = compaction_snapshot_report(&report, 30); + assert!( + snapshot + .reasons + .contains(&TableCompactionPlanningReason::MissingCurrentSnapshot) + ); +} + +#[tokio::test] +async fn compaction_commit_rewrites_small_data_files_and_advances_pointer() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let data_dir = default_table_data_dir_path(&namespace, &table); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + let manifest_list = format!("{metadata_dir}/snap-20.avro"); + let manifest = format!("{metadata_dir}/manifest-20.avro"); + let left_data = format!("{data_dir}/part-left.parquet"); + let right_data = format!("{data_dir}/part-right.parquet"); + let retained_data = format!("{data_dir}/part-retained.parquet"); + let left_parquet = parquet_i32_bytes(&[1, 2]); + let right_parquet = parquet_i32_bytes(&[3, 4]); + let retained_values = (10..20_000).collect::>(); + let retained_parquet = parquet_i32_bytes(&retained_values); + let small_file_threshold_bytes = u64::try_from(left_parquet.len().max(right_parquet.len())).unwrap(); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object( + bucket, + &manifest, + manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0), (&retained_data, 0)]), + ) + .await; + backend.seed_object(bucket, &left_data, left_parquet).await; + backend.seed_object(bucket, &right_data, right_parquet).await; + backend.seed_object(bucket, &retained_data, retained_parquet).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://analytics/tables/table-id", + "last-sequence-number": 7, + "last-updated-ms": 2000, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 2000, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + ], + "current-snapshot-id": 20, + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .commit_table_compaction( + bucket, + "sales", + "orders", + TableCompactionPlanningConfig { + target_file_size_bytes: 64 * 1024, + small_file_threshold_bytes, + min_input_files: 2, + max_rewrite_bytes_per_job: 128 * 1024, + }, + ) + .await + .expect("compaction rewrite should commit"); + + assert_eq!(report.status, TableCompactionPlanningStatus::Committed); + assert_eq!(report.candidate_file_count, 2); + assert_eq!(report.rewrite_group_count, 1); + assert_eq!(report.manual_review_count, 0); + let committed_metadata = report + .committed_metadata_location + .as_ref() + .expect("compaction should report committed metadata"); + assert_ne!(committed_metadata, ¤t); + let rewrite_group = report.rewrite_groups.first().expect("rewrite group should be reported"); + assert_eq!(rewrite_group.input_file_locations, vec![left_data.clone(), right_data.clone()]); + let output_file = rewrite_group + .output_file_location + .as_ref() + .expect("rewrite group should include output data file"); + assert!(output_file.starts_with("s3://analytics/tables/table-id/data/")); + let output_file_key = + table_catalog_object_key_from_location(bucket, output_file).expect("output file should be inside the table bucket"); + assert!(output_file_key.starts_with("tables/table-id/data/")); + let output_object = backend + .read_object(bucket, &output_file_key) + .await + .unwrap() + .expect("compacted data file should be written"); + assert_eq!(parquet_i32_values(output_object.data), vec![1, 2, 3, 4]); + assert!(backend.object_exists(bucket, &left_data).await.unwrap()); + assert!(backend.object_exists(bucket, &right_data).await.unwrap()); + assert!(backend.object_exists(bucket, &retained_data).await.unwrap()); + + let table_entry = store + .load_table(bucket, "sales", "orders") + .await + .unwrap() + .expect("table should still exist"); + assert_eq!(table_entry.metadata_location, *committed_metadata); + assert_eq!(table_entry.generation, 2); + let metadata_object = backend + .read_object(bucket, committed_metadata) + .await + .unwrap() + .expect("compaction metadata should be written"); + let metadata = serde_json::from_slice::(&metadata_object.data).unwrap(); + assert_ne!(metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64), Some(20)); + assert_eq!(metadata.get("snapshots").and_then(serde_json::Value::as_array).unwrap().len(), 2); + assert_eq!( + metadata + .get("metadata-log") + .and_then(serde_json::Value::as_array) + .unwrap() + .last() + .and_then(|entry| entry.get("metadata-file")) + .and_then(serde_json::Value::as_str), + Some(current.as_str()) + ); + assert_eq!( + metadata + .get("snapshot-log") + .and_then(serde_json::Value::as_array) + .unwrap() + .last() + .and_then(|entry| entry.get("snapshot-id")) + .and_then(serde_json::Value::as_i64), + metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64) + ); + let current_manifest_list = metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .unwrap() + .last() + .and_then(|snapshot| snapshot.get("manifest-list")) + .and_then(serde_json::Value::as_str) + .unwrap(); + let manifest_list_object = backend + .read_object(bucket, current_manifest_list) + .await + .unwrap() + .expect("compaction manifest list should be written"); + let manifest_paths = manifest_paths_from_manifest_list_avro(&manifest_list_object.data).unwrap(); + assert_eq!(manifest_paths.len(), 1); + let manifest_object = backend + .read_object(bucket, &manifest_paths[0]) + .await + .unwrap() + .expect("compaction manifest should be written"); + let manifest_references = file_references_from_manifest_avro(&manifest_object.data).unwrap(); + let manifest_data_files = manifest_references + .into_iter() + .filter_map(|(location, kind)| (kind == TableMetadataMaintenanceObjectKind::DataFile).then_some(location)) + .collect::>(); + assert!(manifest_data_files.contains(output_file)); + assert!(manifest_data_files.contains(&retained_data)); + assert!(!manifest_data_files.contains(&left_data)); + assert!(!manifest_data_files.contains(&right_data)); +} + +#[tokio::test] +async fn compaction_commit_keeps_partition_rewrite_groups_isolated() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let data_dir = "tables/table-id/data"; + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + let manifest_list = format!("{metadata_dir}/snap-20.avro"); + let manifest = format!("{metadata_dir}/manifest-20.avro"); + let left_data = format!("{data_dir}/dt=2026-06-24/part-left.parquet"); + let right_data = format!("{data_dir}/dt=2026-06-24/part-right.parquet"); + let other_partition_data = format!("{data_dir}/dt=2026-06-25/part-only.parquet"); + let left_parquet = parquet_i32_bytes(&[1, 2]); + let right_parquet = parquet_i32_bytes(&[3, 4]); + let other_partition_parquet = parquet_i32_bytes(&[5, 6]); + let small_file_threshold_bytes = + u64::try_from(left_parquet.len().max(right_parquet.len()).max(other_partition_parquet.len())).unwrap(); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object( + bucket, + &manifest, + manifest_avro_bytes_with_dt_partition(&[ + (&left_data, 0, "2026-06-24"), + (&right_data, 0, "2026-06-24"), + (&other_partition_data, 0, "2026-06-25"), + ]), + ) + .await; + backend.seed_object(bucket, &left_data, left_parquet).await; + backend.seed_object(bucket, &right_data, right_parquet).await; + backend + .seed_object(bucket, &other_partition_data, other_partition_parquet) + .await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://analytics/tables/table-id", + "last-sequence-number": 7, + "last-updated-ms": 2000, + "schemas": [ + { + "schema-id": 0, + "type": "struct", + "fields": [ + {"id": 1, "name": "id", "required": true, "type": "int"}, + {"id": 2, "name": "dt", "required": false, "type": "string"} + ] + } + ], + "current-schema-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [ + { + "source-id": 2, + "field-id": 1000, + "name": "dt", + "transform": "identity" + } + ] + } + ], + "default-spec-id": 0, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 2000, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + ], + "current-snapshot-id": 20, + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .commit_table_compaction( + bucket, + "sales", + "orders", + TableCompactionPlanningConfig { + target_file_size_bytes: 64 * 1024, + small_file_threshold_bytes, + min_input_files: 2, + max_rewrite_bytes_per_job: 128 * 1024, + }, + ) + .await + .expect("partition-local compaction rewrite should commit"); + + assert_eq!(report.status, TableCompactionPlanningStatus::Committed); + assert_eq!(report.candidate_file_count, 3); + assert_eq!(report.rewrite_group_count, 1); + let rewrite_group = report.rewrite_groups.first().expect("rewrite group should be reported"); + assert_eq!(rewrite_group.input_file_locations, vec![left_data.clone(), right_data.clone()]); + let output_file = rewrite_group + .output_file_location + .as_ref() + .expect("rewrite group should include output data file"); + assert!(output_file.starts_with("s3://analytics/tables/table-id/data/dt=2026-06-24/")); + let output_file_key = + table_catalog_object_key_from_location(bucket, output_file).expect("output file should be inside the table bucket"); + let output_object = backend + .read_object(bucket, &output_file_key) + .await + .unwrap() + .expect("compacted partition data file should be written"); + assert_eq!(parquet_i32_values(output_object.data), vec![1, 2, 3, 4]); + assert!(backend.object_exists(bucket, &other_partition_data).await.unwrap()); + + let table_entry = store + .load_table(bucket, "sales", "orders") + .await + .unwrap() + .expect("table should still exist"); + let metadata_object = backend + .read_object(bucket, &table_entry.metadata_location) + .await + .unwrap() + .expect("compaction metadata should be written"); + let metadata = serde_json::from_slice::(&metadata_object.data).unwrap(); + let current_manifest_list = metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .unwrap() + .last() + .and_then(|snapshot| snapshot.get("manifest-list")) + .and_then(serde_json::Value::as_str) + .unwrap(); + let manifest_list_object = backend + .read_object(bucket, current_manifest_list) + .await + .unwrap() + .expect("compaction manifest list should be written"); + let manifest_references = manifest_list_references_from_manifest_list_avro(&manifest_list_object.data).unwrap(); + assert_eq!(manifest_references.len(), 1); + assert_eq!(manifest_references[0].partition_spec_id, Some(0)); + let manifest_object = backend + .read_object(bucket, &manifest_references[0].manifest_path) + .await + .unwrap() + .expect("compaction manifest should be written"); + let data_file_references = data_file_references_from_manifest_avro(&manifest_object.data).unwrap(); + let output_reference = data_file_references + .iter() + .find(|reference| reference.location == *output_file) + .expect("compacted output should be present in the manifest"); + assert_eq!( + output_reference.partition, + vec![("dt".to_string(), apache_avro::types::Value::String("2026-06-24".to_string()))] + ); + let retained_reference = data_file_references + .iter() + .find(|reference| reference.location == other_partition_data) + .expect("retained partition file should stay in the manifest"); + assert_eq!( + retained_reference.partition, + vec![("dt".to_string(), apache_avro::types::Value::String("2026-06-25".to_string()))] + ); +} + +#[tokio::test] +async fn compaction_commit_preserves_sort_order_and_keeps_groups_isolated() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let data_dir = default_table_data_dir_path(&namespace, &table); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + let manifest_list = format!("{metadata_dir}/snap-20.avro"); + let manifest = format!("{metadata_dir}/manifest-20.avro"); + let left_data = format!("{data_dir}/part-left.parquet"); + let right_data = format!("{data_dir}/part-right.parquet"); + let other_sort_data = format!("{data_dir}/part-other-sort.parquet"); + let left_parquet = parquet_i32_bytes(&[1, 2]); + let right_parquet = parquet_i32_bytes(&[3, 4]); + let other_sort_parquet = parquet_i32_bytes(&[5, 6]); + let small_file_threshold_bytes = + u64::try_from(left_parquet.len().max(right_parquet.len()).max(other_sort_parquet.len())).unwrap(); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object( + bucket, + &manifest, + manifest_avro_bytes_with_sort_order(&[(&left_data, 0, 7), (&right_data, 0, 7), (&other_sort_data, 0, 8)]), + ) + .await; + backend.seed_object(bucket, &left_data, left_parquet).await; + backend.seed_object(bucket, &right_data, right_parquet).await; + backend.seed_object(bucket, &other_sort_data, other_sort_parquet).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://analytics/tables/table-id", + "last-sequence-number": 7, + "last-updated-ms": 2000, + "sort-orders": [ + {"order-id": 7, "fields": []}, + {"order-id": 8, "fields": []} + ], + "default-sort-order-id": 7, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 2000, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + ], + "current-snapshot-id": 20, + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + })) + .unwrap(), + ) + .await; + + let report = store + .commit_table_compaction( + bucket, + "sales", + "orders", + TableCompactionPlanningConfig { + target_file_size_bytes: 64 * 1024, + small_file_threshold_bytes, + min_input_files: 2, + max_rewrite_bytes_per_job: 128 * 1024, + }, + ) + .await + .expect("sort-aware compaction rewrite should commit"); + + assert_eq!(report.status, TableCompactionPlanningStatus::Committed); + assert_eq!(report.candidate_file_count, 3); + assert_eq!(report.rewrite_group_count, 1); + let rewrite_group = report.rewrite_groups.first().expect("rewrite group should be reported"); + assert_eq!(rewrite_group.sort_order_id, Some(7)); + assert_eq!(rewrite_group.input_file_locations, vec![left_data.clone(), right_data.clone()]); + let output_file = rewrite_group + .output_file_location + .as_ref() + .expect("rewrite group should include output data file"); + + let table_entry = store + .load_table(bucket, "sales", "orders") + .await + .unwrap() + .expect("table should still exist"); + let metadata_object = backend + .read_object(bucket, &table_entry.metadata_location) + .await + .unwrap() + .expect("compaction metadata should be written"); + let metadata = serde_json::from_slice::(&metadata_object.data).unwrap(); + let current_manifest_list = metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .unwrap() + .last() + .and_then(|snapshot| snapshot.get("manifest-list")) + .and_then(serde_json::Value::as_str) + .unwrap(); + let manifest_list_object = backend + .read_object(bucket, current_manifest_list) + .await + .unwrap() + .expect("compaction manifest list should be written"); + let manifest_references = manifest_list_references_from_manifest_list_avro(&manifest_list_object.data).unwrap(); + let manifest_object = backend + .read_object(bucket, &manifest_references[0].manifest_path) + .await + .unwrap() + .expect("compaction manifest should be written"); + let data_file_references = data_file_references_from_manifest_avro(&manifest_object.data).unwrap(); + let output_reference = data_file_references + .iter() + .find(|reference| reference.location == *output_file) + .expect("compacted output should be present in the manifest"); + assert_eq!(output_reference.sort_order_id, Some(7)); + let retained_reference = data_file_references + .iter() + .find(|reference| reference.location == other_sort_data) + .expect("different sort order file should stay in the manifest"); + assert_eq!(retained_reference.sort_order_id, Some(8)); +} + +#[tokio::test] +async fn compaction_commit_rejects_schema_mismatch_without_advancing_pointer() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let data_dir = default_table_data_dir_path(&namespace, &table); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + let manifest_list = format!("{metadata_dir}/snap-20.avro"); + let manifest = format!("{metadata_dir}/manifest-20.avro"); + let left_data = format!("{data_dir}/part-left.parquet"); + let right_data = format!("{data_dir}/part-right.parquet"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0)])) + .await; + backend.seed_object(bucket, &left_data, parquet_i32_bytes(&[1, 2])).await; + backend.seed_object(bucket, &right_data, parquet_i64_bytes(&[3, 4])).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://analytics/tables/table-id", + "last-sequence-number": 7, + "last-updated-ms": 2000, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 2000, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } + } + ], + "current-snapshot-id": 20 + })) + .unwrap(), + ) + .await; + + let error = store + .commit_table_compaction( + bucket, + "sales", + "orders", + TableCompactionPlanningConfig { + target_file_size_bytes: 64 * 1024, + small_file_threshold_bytes: 32 * 1024, + min_input_files: 2, + max_rewrite_bytes_per_job: 128 * 1024, + }, + ) + .await + .expect_err("schema mismatch should reject compaction commit"); + + assert!(matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("schemas must match"))); + let table_entry = store + .load_table(bucket, "sales", "orders") + .await + .unwrap() + .expect("table should still exist"); + assert_eq!(table_entry.metadata_location, current); +} + +#[tokio::test] +async fn compaction_commit_rejects_deleted_manifest_entries_without_advancing_pointer() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_dir = default_table_metadata_dir_path(&namespace, &table); + let data_dir = default_table_data_dir_path(&namespace, &table); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + let manifest_list = format!("{metadata_dir}/snap-20.avro"); + let manifest = format!("{metadata_dir}/manifest-20.avro"); + let left_data = format!("{data_dir}/part-left.parquet"); + let deleted_data = format!("{data_dir}/part-deleted.parquet"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) + .await; + backend + .seed_object( + bucket, + &manifest, + manifest_avro_bytes_with_status(&[(&left_data, 0, 1), (&deleted_data, 0, 2)]), + ) + .await; + backend.seed_object(bucket, &left_data, parquet_i32_bytes(&[1, 2])).await; + backend.seed_object(bucket, &deleted_data, parquet_i32_bytes(&[3, 4])).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://analytics/tables/table-id", + "last-sequence-number": 7, + "last-updated-ms": 2000, + "metadata-log": [], + "snapshots": [ + { + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 2000, + "manifest-list": manifest_list, + "summary": { + "operation": "overwrite" + } + } + ], + "current-snapshot-id": 20 + })) + .unwrap(), + ) + .await; + + let error = store + .commit_table_compaction( + bucket, + "sales", + "orders", + TableCompactionPlanningConfig { + target_file_size_bytes: 64 * 1024, + small_file_threshold_bytes: 32 * 1024, + min_input_files: 2, + max_rewrite_bytes_per_job: 128 * 1024, + }, + ) + .await + .expect_err("deleted manifest entries should reject compaction commit"); + + assert!(matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("no safe rewrite candidates"))); + let table_entry = store + .load_table(bucket, "sales", "orders") + .await + .unwrap() + .expect("table should still exist"); + assert_eq!(table_entry.metadata_location, current); +} + +#[tokio::test] +async fn maintenance_dry_run_keeps_recent_metadata_files_and_ignores_non_metadata_objects() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let v1 = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let v2 = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let recent = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00004.metadata.json"); + let manifest = format!("{}/snap-1.avro", default_table_metadata_dir_path(&namespace, &table)); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + for metadata in [&v1, &v2, &recent] { + backend.seed_object(bucket, metadata, b"{}".to_vec()).await; + } + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + backend.seed_object(bucket, &manifest, b"manifest".to_vec()).await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 2) + .await + .unwrap(); + + assert!(report.retained_metadata_locations.contains(&recent)); + assert!(report.retained_metadata_locations.contains(¤t)); + assert_eq!(report.cleanup_candidate_locations, vec![v1, v2]); + assert!(!report.cleanup_candidate_locations.contains(&manifest)); +} + +#[tokio::test] +async fn maintenance_delete_removes_only_dry_run_metadata_candidates() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let retained = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + let manifest = format!("{}/snap-1.avro", default_table_metadata_dir_path(&namespace, &table)); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend.seed_object(bucket, &retained, b"{}".to_vec()).await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "metadata-log": [ + { + "timestamp-ms": 1, + "metadata-file": retained + } + ], + "snapshots": [ + { + "snapshot-id": 1, + "manifest-list": manifest + } + ] + })) + .unwrap(), + ) + .await; + backend.seed_object(bucket, &manifest, manifest_list_avro_bytes(&[])).await; + + let report = store + .delete_table_metadata_maintenance_candidates(bucket, "sales", "orders", 0) + .await + .unwrap(); + + assert_eq!(report.cleanup_candidate_locations, vec![old.clone()]); + assert!(report.cleanup_object_candidate_locations.is_empty()); + assert!(!backend.object_exists(bucket, &old).await.unwrap()); + assert!(backend.object_exists(bucket, &retained).await.unwrap()); + assert!(backend.object_exists(bucket, ¤t).await.unwrap()); + assert!(backend.object_exists(bucket, &manifest).await.unwrap()); +} + +#[tokio::test] +async fn maintenance_delete_skips_recent_uncommitted_metadata_candidates() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let fresh = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + backend + .seed_object_with_mod_time(bucket, &fresh, br#"{"metadata-log":[]}"#.to_vec(), Some(OffsetDateTime::now_utc())) + .await; + + let report = store + .delete_table_metadata_maintenance_candidates(bucket, "sales", "orders", 0) + .await + .unwrap(); + + assert_eq!(report.cleanup_candidate_locations, vec![old.clone()]); + assert!(!backend.object_exists(bucket, &old).await.unwrap()); + assert!(backend.object_exists(bucket, &fresh).await.unwrap()); +} + +#[tokio::test] +async fn maintenance_delete_does_not_expand_beyond_planned_deletable_candidates() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let fresh = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + backend + .seed_object_with_mod_time(bucket, &fresh, b"{}".to_vec(), Some(OffsetDateTime::now_utc())) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + assert_eq!(report.cleanup_candidate_locations, vec![old.clone(), fresh.clone()]); + assert_eq!(report.deletable_metadata_locations, vec![old.clone()]); + + backend.seed_object(bucket, &fresh, b"{}".to_vec()).await; + let deleted = store + .delete_table_metadata_maintenance_report(bucket, "sales", "orders", report) + .await + .unwrap(); + + assert_eq!(deleted.job.operation, TableMetadataMaintenanceOperation::Delete); + assert_eq!(deleted.job.status, TableMetadataMaintenanceJobStatus::Successful); + assert_eq!(deleted.job.deleted_metadata_file_count, 1); + assert_eq!(deleted.cleanup_candidate_locations, vec![old.clone()]); + assert_eq!(deleted.deletable_metadata_locations, vec![old.clone()]); + assert_eq!( + maintenance_object_report(&deleted, &old).state, + TableMetadataMaintenanceObjectState::Deleted + ); + assert_eq!( + maintenance_object_report(&deleted, &fresh).state, + TableMetadataMaintenanceObjectState::PendingSafetyWindow + ); + assert!(!backend.object_exists(bucket, &old).await.unwrap()); + assert!(backend.object_exists(bucket, &fresh).await.unwrap()); +} + +#[tokio::test] +async fn maintenance_delete_conflicts_when_current_pointer_changes_before_delete() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let old = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let next = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &old, b"{}".to_vec()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + assert_eq!(report.cleanup_candidate_locations, vec![old.clone()]); + + backend.seed_object(bucket, &next, br#"{"metadata-log":[]}"#.to_vec()).await; + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-id".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current, + new_metadata_location: next, + requirements: Vec::new(), + writer: Some("test".to_string()), + }) + .await + .unwrap(); + + let err = store + .delete_table_metadata_maintenance_report(bucket, "sales", "orders", report) + .await + .unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Conflict(_)); + assert!(backend.object_exists(bucket, &old).await.unwrap()); +} + +#[tokio::test] +async fn export_catalog_entry_contains_table_identity_and_pointer() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + + let export = store.export_table_catalog_entry(bucket, "sales", "orders").await.unwrap(); + + assert_eq!(export.table_bucket.table_bucket, bucket); + assert_eq!(export.namespace.namespace, "sales"); + assert_eq!(export.table.namespace, "sales"); + assert_eq!(export.table.table, "orders"); + assert_eq!(export.table.table_id, "table-id"); + assert_eq!(export.table.table_uuid, "table-uuid"); + assert_eq!(export.table.metadata_location, current); + assert_eq!(export.table.version_token, "token-v1"); + assert_eq!(export.table.generation, 1); +} + +#[tokio::test] +async fn export_catalog_entry_includes_backing_migration_manifest() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + + let export = store.export_table_catalog_entry(bucket, "sales", "orders").await.unwrap(); + + assert_eq!(export.backing_manifest.version, TABLE_CATALOG_BACKING_MANIFEST_VERSION); + assert_eq!(export.backing_manifest.current.kind, TableCatalogBackingKind::ObjectBacked); + assert_eq!(export.backing_manifest.current.authority, TableCatalogAuthority::RustfsSysObject); + assert_eq!( + export.backing_manifest.current.consistency, + TableCatalogConsistencyMode::ConditionalObjectCas + ); + assert_eq!(export.backing_manifest.current.wal.finalization_required_count, 0); + assert_eq!(export.backing_manifest.migration.target_kind, TableCatalogBackingKind::StrongKvWal); + assert_eq!( + export.backing_manifest.migration.status, + TableCatalogBackingMigrationStatus::ReadyToSnapshot + ); + assert!( + export + .backing_manifest + .migration + .required_steps + .contains(&TableCatalogBackingMigrationStep::ReplayCommitLog) + ); + assert_eq!( + export.backing_manifest.ha.writer_region_model, + TableCatalogHaWriterModel::SingleActiveWriterRegion + ); + assert!(!export.backing_manifest.ha.active_active_supported); +} + +#[tokio::test] +async fn object_table_catalog_store_serializes_namespace_drop_with_table_creation() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + + let table_path = store.paths.table_entry_path(bucket, &namespace, &table); + let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &table_path).await; + let create_store = store.clone(); + let table_entry = test_table_entry(bucket, &namespace, &table, metadata_location); + let create = tokio::spawn(async move { create_store.create_table(table_entry).await }); + pause.wait_started().await; + + let drop_store = store.clone(); + let namespace_name = namespace.public_name(); + let mut drop_namespace = tokio::spawn(async move { drop_store.drop_namespace(bucket, &namespace_name).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut drop_namespace) + .await + .is_err() + ); + + pause.release(); + create.await.unwrap().unwrap(); + assert_matches!( + drop_namespace.await.unwrap(), + Err(TableCatalogStoreError::Conflict(message)) if message.contains("is not empty") + ); + assert!( + store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn durable_strong_migration_dry_run_reports_ready_catalog_inventory() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let view = IdentifierSegment::parse("recent_orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let view_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata.clone()).await; + store + .create_view(test_view_entry(bucket, &namespace, &view, view_metadata)) + .await + .unwrap(); + backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap(); + + let report = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); + + assert_eq!(report.table_bucket, bucket); + assert_eq!(report.source_kind, TableCatalogBackingKind::ObjectBacked); + assert_eq!(report.target_kind, TableCatalogBackingKind::StrongKvWal); + assert_eq!(report.status, TableCatalogBackingMigrationStatus::ReadyToSnapshot); + assert_eq!(report.namespace_count, 1); + assert_eq!(report.table_count, 1); + assert_eq!(report.view_count, 1); + assert_eq!(report.commit_log_count, 1); + assert_eq!(report.idempotency_index_count, 1); + assert_eq!(report.warehouse_prefix_count, 1); + assert!(!report.object_backed_writes_fenced); + assert!(!report.ready_to_enable_durable_strong); + assert!(report.blockers.is_empty()); + assert!( + report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::SnapshotObjectBackedCatalog) + ); + assert!( + !report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::EnableDurableStrongBacking) + ); + assert_eq!(report.rollback.backing_config_key, ENV_TABLE_CATALOG_BACKING); + assert_eq!(report.rollback.rollback_backing_value, TABLE_CATALOG_BACKING_OBJECT); +} + +#[tokio::test] +async fn durable_strong_migration_rejects_orphan_table_without_namespace() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; + + let namespace_path = object_store.paths.namespace_entry_path(bucket, &namespace); + backend + .delete_object(RUSTFS_META_BUCKET, &namespace_path) + .await + .expect("namespace marker should be removed"); + assert!( + object_store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .unwrap() + .is_some() + ); + + let error = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap_err(); + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("table entry has no namespace entry") + ); + object_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("failed migration must leave source catalog writable"); + object_store.put_table_bucket(test_bucket_entry("research")).await.unwrap(); +} + +#[tokio::test] +async fn durable_strong_migration_rejects_orphan_view_without_namespace() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let metadata_location = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + + object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + object_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + object_store + .create_view(test_view_entry(bucket, &namespace, &view, metadata_location)) + .await + .unwrap(); + object_store.backfill_table_warehouse_index(bucket).await.unwrap(); + + let namespace_path = object_store.paths.namespace_entry_path(bucket, &namespace); + backend + .delete_object(RUSTFS_META_BUCKET, &namespace_path) + .await + .expect("namespace marker should be removed"); + assert!( + object_store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .unwrap() + .is_some() + ); + + let error = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap_err(); + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("view entry has no namespace entry") + ); + object_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("failed migration must leave source catalog writable"); +} + +#[tokio::test] +async fn durable_strong_migration_rejects_orphan_idempotency_without_fencing_source() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata.clone()).await; + + let orphan = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "orphan-commit".to_string(), + idempotency_key: Some("orphan-request".to_string()), + table_id: "table-id".to_string(), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: current_metadata.clone(), + new_metadata_location: current_metadata, + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: None, + created_at: None, + updated_at: None, + }; + let idempotency_path = object_store + .paths + .commit_idempotency_entry_path(bucket, "table-id", "orphan-request"); + backend + .seed_object(RUSTFS_META_BUCKET, &idempotency_path, serde_json::to_vec(&orphan).unwrap()) + .await; + + let error = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap_err(); + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("has no commit record") + ); + + object_store + .create_namespace(test_namespace_entry(bucket, &Namespace::parse("still_writable").unwrap())) + .await + .unwrap(); + object_store.put_table_bucket(test_bucket_entry("research")).await.unwrap(); +} + +#[tokio::test] +async fn durable_strong_migration_materializes_catalog_state_and_fences_object_writes() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let view = IdentifierSegment::parse("recent_orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let strong_metadata = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + let view_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata.clone()).await; + object_store + .create_view(test_view_entry(bucket, &namespace, &view, view_metadata)) + .await + .unwrap(); + backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; + let object_commit = object_store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "object-commit".to_string(), + idempotency_key: Some("object-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata.clone(), + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap(); + + let materialized = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap(); + assert_eq!(materialized.status, TableCatalogBackingMigrationExecutionStatus::SnapshotMaterialized); + assert_eq!(materialized.namespace_count, 1); + assert_eq!(materialized.table_count, 1); + assert_eq!(materialized.view_count, 1); + assert_eq!(materialized.commit_log_count, 1); + assert_eq!(materialized.idempotency_index_count, 1); + assert!(materialized.object_backed_writes_fenced); + assert!(materialized.ready_to_enable_durable_strong); + + let retry = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap(); + assert_eq!(retry.status, TableCatalogBackingMigrationExecutionStatus::SnapshotAlreadyMaterialized); + assert_eq!(retry.source_fingerprint, materialized.source_fingerprint); + + let migration = object_store.plan_durable_strong_backing_migration(bucket).await.unwrap(); + assert_eq!(migration.status, TableCatalogBackingMigrationStatus::SnapshotMaterialized); + assert!(migration.object_backed_writes_fenced); + assert!(migration.ready_to_enable_durable_strong); + assert!( + migration + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::EnableDurableStrongBacking) + ); + + let strong_store = StrongTableCatalogStore::new(backend.clone()); + let migrated_table = strong_store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(migrated_table.metadata_location, next_metadata); + assert_eq!(migrated_table.version_token, object_commit.table.version_token); + assert_eq!(migrated_table.generation, 2); + assert!( + strong_store + .load_view(bucket, "sales", "recent_orders") + .await + .unwrap() + .is_some() + ); + assert!( + strong_store + .get_commit_by_id(bucket, "table-id", "object-commit") + .await + .unwrap() + .is_some() + ); + assert!( + strong_store + .get_commit_by_idempotency_key(bucket, "table-id", "object-request") + .await + .unwrap() + .is_some() + ); + + let object_err = object_store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "stale-object-commit".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: migrated_table.version_token.clone(), + expected_metadata_location: migrated_table.metadata_location.clone(), + new_metadata_location: strong_metadata.clone(), + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap_err(); + assert_matches!(object_err, TableCatalogStoreError::Conflict(message) if message.contains("writes are fenced")); + + backend.seed_object(bucket, &strong_metadata, b"{}".to_vec()).await; + let strong_commit = strong_store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "strong-commit".to_string(), + idempotency_key: Some("strong-request".to_string()), + operation: "append".to_string(), + expected_version_token: migrated_table.version_token, + expected_metadata_location: migrated_table.metadata_location, + new_metadata_location: strong_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap(); + assert_eq!(strong_commit.table.generation, 3); + + let cancel_err = object_store + .cancel_durable_strong_backing_migration(bucket) + .await + .unwrap_err(); + assert_matches!(cancel_err, TableCatalogStoreError::Conflict(message) if message.contains("changed after materializing")); +} + +#[tokio::test] +async fn durable_strong_migration_cancel_restores_object_backed_writes_before_target_changes() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata.clone()).await; + object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap(); + + let cancelled = object_store.cancel_durable_strong_backing_migration(bucket).await.unwrap(); + assert_eq!(cancelled.status, TableCatalogBackingMigrationCancelStatus::FenceReleased); + assert!(!cancelled.object_backed_writes_fenced); + assert!( + StrongTableCatalogStore::new(backend.clone()) + .load_table(bucket, "sales", "orders") + .await + .unwrap() + .is_none() + ); + object_store.put_table_bucket(test_bucket_entry("research")).await.unwrap(); + + backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; + let result = object_store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "rollback-object-commit".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap(); + assert_eq!(result.table.generation, 2); +} + +#[tokio::test] +async fn durable_strong_migration_waits_for_in_flight_catalog_writes() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + object_store.backfill_table_warehouse_index(bucket).await.unwrap(); + + let namespace_path = object_store.paths.namespace_entry_path(bucket, &namespace); + let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &namespace_path).await; + let namespace_store = object_store.clone(); + let namespace_write = tokio::spawn(async move { + namespace_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + }); + pause.wait_started().await; + + let migration_store = object_store.clone(); + let mut migration = tokio::spawn(async move { migration_store.materialize_durable_strong_backing_migration(bucket).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut migration) + .await + .is_err() + ); + + pause.release(); + namespace_write.await.unwrap().unwrap(); + let report = migration.await.unwrap().unwrap(); + assert_eq!(report.namespace_count, 1); + assert!( + StrongTableCatalogStore::new(backend) + .get_namespace(bucket, "sales") + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn durable_strong_migration_retry_recovers_after_fence_finalization_failure() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; + + let fence_path = object_store.paths.backing_migration_fence_path(bucket); + backend.fail_put_attempt(RUSTFS_META_BUCKET, &fence_path, 2).await; + let first = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap_err(); + assert_matches!(first, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); + assert_matches!( + object_store + .create_namespace(test_namespace_entry(bucket, &Namespace::parse("blocked").unwrap())) + .await + .unwrap_err(), + TableCatalogStoreError::Conflict(message) if message.contains("writes are fenced") + ); + + let retry = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap(); + assert_eq!(retry.status, TableCatalogBackingMigrationExecutionStatus::SnapshotAlreadyMaterialized); + assert!(retry.ready_to_enable_durable_strong); +} + +#[tokio::test] +async fn durable_strong_migration_cancel_preserves_preexisting_target_state() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; + + let strong_store = StrongTableCatalogStore::new(backend.clone()); + strong_store + .put_table_bucket(object_store.get_table_bucket(bucket).await.unwrap().unwrap()) + .await + .unwrap(); + strong_store + .create_namespace(object_store.get_namespace(bucket, "sales").await.unwrap().unwrap()) + .await + .unwrap(); + strong_store + .create_table(object_store.load_table(bucket, "sales", "orders").await.unwrap().unwrap()) + .await + .unwrap(); + + let materialized = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap(); + assert_eq!( + materialized.status, + TableCatalogBackingMigrationExecutionStatus::SnapshotAlreadyMaterialized + ); + object_store.cancel_durable_strong_backing_migration(bucket).await.unwrap(); + assert!(strong_store.load_table(bucket, "sales", "orders").await.unwrap().is_some()); +} + +#[tokio::test] +async fn durable_strong_migration_requires_every_table_bucket_before_cutover() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend); + for bucket in ["analytics", "research"] { + object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + object_store.backfill_table_warehouse_index(bucket).await.unwrap(); + } + + let first = object_store + .materialize_durable_strong_backing_migration("analytics") + .await + .unwrap(); + assert!(!first.ready_to_enable_durable_strong); + assert_matches!( + object_store + .put_table_bucket(test_bucket_entry("new-bucket")) + .await + .unwrap_err(), + TableCatalogStoreError::Conflict(message) if message.contains("registry writes are fenced") + ); + assert!( + object_store + .plan_durable_strong_backing_migration("analytics") + .await + .unwrap() + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::SnapshotRemainingTableBuckets) + ); + + let second = object_store + .materialize_durable_strong_backing_migration("research") + .await + .unwrap(); + assert!(second.ready_to_enable_durable_strong); + assert!( + object_store + .plan_durable_strong_backing_migration("analytics") + .await + .unwrap() + .ready_to_enable_durable_strong + ); +} + +#[tokio::test] +async fn durable_strong_migration_dry_run_reports_recovery_blockers() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata.clone()).await; + backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap(); + let idempotency_path = store + .paths + .commit_idempotency_entry_path(bucket, "table-id", "client-request"); + backend.delete_object(RUSTFS_META_BUCKET, &idempotency_path).await.unwrap(); + + let report = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); + + assert_eq!(report.status, TableCatalogBackingMigrationStatus::RecoveryRequired); + assert_eq!(report.commit_log_count, 1); + assert_eq!(report.idempotency_index_count, 0); + assert!( + report + .blockers + .contains(&TableCatalogBackingMigrationBlocker::CommitRecoveryRequired) + ); + assert!( + report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::RunCatalogRecovery) + ); + assert!( + !report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::SnapshotObjectBackedCatalog) + ); + assert!( + !report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::EnableDurableStrongBacking) + ); + assert!( + !report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::VerifyDurableStrongSnapshot) + ); +} + +#[tokio::test] +async fn durable_strong_migration_dry_run_requires_ready_warehouse_index() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata).await; + let state_path = store.paths.warehouse_index_state_path(bucket); + backend.delete_object(RUSTFS_META_BUCKET, &state_path).await.unwrap(); + + let report = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); + + assert_eq!(report.status, TableCatalogBackingMigrationStatus::RecoveryRequired); + assert!(!report.warehouse_index_ready); + assert!( + report + .blockers + .contains(&TableCatalogBackingMigrationBlocker::WarehouseIndexBackfillRequired) + ); + assert!( + report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::BackfillWarehouseIndex) + ); + assert!( + !report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::SnapshotObjectBackedCatalog) + ); + assert!( + !report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::EnableDurableStrongBacking) + ); + assert!( + !report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::VerifyDurableStrongSnapshot) + ); +} + +#[tokio::test] +async fn strong_catalog_backing_commit_is_atomic_with_wal_and_idempotency() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + let result = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata.clone(), + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap(); + + assert_eq!(result.table.metadata_location, new_metadata); + assert_eq!(result.table.generation, 2); + assert_eq!(result.commit_log.status, CommitLogStatus::Committed); + + let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(loaded.metadata_location, result.table.metadata_location); + assert_eq!(loaded.version_token, result.table.version_token); + assert_eq!( + store + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .unwrap() + .unwrap() + .status, + CommitLogStatus::Committed + ); + assert_eq!( + store + .get_commit_by_idempotency_key(bucket, "table-id", "client-request") + .await + .unwrap() + .unwrap() + .status, + CommitLogStatus::Committed + ); + + let recovery = store.plan_table_commit_recovery(bucket, "sales", "orders").await.unwrap(); + assert_eq!(recovery.staged_before_table_update_count, 0); + assert_eq!(recovery.finalization_required_count, 0); + assert_eq!(recovery.idempotency_repair_required_count, 0); + assert_eq!(recovery.manual_review_count, 0); +} + +#[tokio::test] +async fn strong_catalog_pagination_uses_ordered_state_and_rejects_object_cursors() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let namespace_name = namespace.public_name(); + let one = NonZeroUsize::new(1).expect("page size should be non-zero"); + + seed_catalog_list_entries(&store, bucket, &namespace).await; + + let first = store + .list_namespaces_page(bucket, None, one) + .await + .expect("first strong namespace page should load"); + assert_eq!(first.entries[0].namespace, "analytics"); + let second = store + .list_namespaces_page(bucket, first.next_cursor.as_deref(), one) + .await + .expect("second strong namespace page should load"); + assert_eq!(second.entries[0].namespace, "sales"); + assert!(second.next_cursor.is_none()); + + let first = store + .list_tables_page(bucket, &namespace_name, None, one) + .await + .expect("first strong table page should load"); + assert_eq!(first.entries[0].table, "alpha"); + assert!(first.next_cursor.as_deref().is_some_and(|cursor| cursor == "strong:alpha")); + let second = store + .list_tables_page(bucket, &namespace_name, first.next_cursor.as_deref(), one) + .await + .expect("second strong table page should load"); + assert_eq!(second.entries[0].table, "beta"); + assert!(second.next_cursor.is_none()); + + let first = store + .list_views_page(bucket, &namespace_name, None, one) + .await + .expect("first strong view page should load"); + assert_eq!(first.entries[0].view, "alpha"); + let second = store + .list_views_page(bucket, &namespace_name, first.next_cursor.as_deref(), one) + .await + .expect("second strong view page should load"); + assert_eq!(second.entries[0].view, "beta"); + assert!(second.next_cursor.is_none()); + + let exact = NonZeroUsize::new(2).expect("exact page size should be non-zero"); + assert!( + store + .list_namespaces_page(bucket, None, exact) + .await + .expect("exact strong namespace page should load") + .next_cursor + .is_none() + ); + assert!( + store + .list_tables_page(bucket, &namespace_name, None, exact) + .await + .expect("exact strong table page should load") + .next_cursor + .is_none() + ); + assert!( + store + .list_views_page(bucket, &namespace_name, None, exact) + .await + .expect("exact strong view page should load") + .next_cursor + .is_none() + ); + + assert!(matches!( + store + .list_tables_page(bucket, &namespace_name, Some("object:alpha"), one) + .await, + Err(TableCatalogStoreError::Invalid(_)) + )); +} + +#[tokio::test] +async fn strong_catalog_backing_replays_durable_commit_state_after_restart() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + let result = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata.clone(), + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .expect("commit should succeed"); + + let restarted = StrongTableCatalogStore::new(backend.clone()); + let loaded = restarted + .load_table(bucket, "sales", "orders") + .await + .expect("table load should succeed") + .expect("table should replay from durable state"); + assert_eq!(loaded.metadata_location, result.table.metadata_location); + assert_eq!(loaded.version_token, result.table.version_token); + assert_eq!( + restarted + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .expect("commit lookup should succeed") + .expect("commit log should replay from durable state") + .status, + CommitLogStatus::Committed + ); + assert_eq!( + restarted + .get_commit_by_idempotency_key(bucket, "table-id", "client-request") + .await + .expect("idempotency lookup should succeed") + .expect("idempotency index should replay from durable state") + .status, + CommitLogStatus::Committed + ); + + let recovery = restarted + .plan_table_commit_recovery(bucket, "sales", "orders") + .await + .expect("recovery report should replay from durable state"); + assert_eq!(recovery.finalized_count, 1); + assert_eq!(recovery.finalization_required_count, 0); + assert_eq!(recovery.idempotency_repair_required_count, 0); + assert_eq!(recovery.manual_review_count, 0); +} + +#[tokio::test] +async fn strong_catalog_backing_rejects_stale_snapshot_cas_after_concurrent_restart() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let first_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let stale_metadata = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &first_metadata, b"{}".to_vec()).await; + backend.seed_object(bucket, &stale_metadata, b"{}".to_vec()).await; + + let stale_store = StrongTableCatalogStore::new(backend.clone()); + stale_store + .load_table(bucket, "sales", "orders") + .await + .expect("stale store should hydrate") + .expect("table should exist"); + + let first_result = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: first_metadata.clone(), + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .expect("first commit should succeed"); + + let err = stale_store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-2".to_string(), + idempotency_key: Some("client-request-2".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: stale_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .expect_err("stale snapshot CAS should fail"); + + assert_matches!(err, TableCatalogStoreError::Conflict(_)); + let loaded = stale_store + .load_table(bucket, "sales", "orders") + .await + .expect("stale store should reload after CAS conflict") + .expect("table should still exist"); + assert_eq!(loaded.metadata_location, first_result.table.metadata_location); + assert_eq!(loaded.version_token, first_result.table.version_token); + assert!( + stale_store + .get_commit_by_id(bucket, "table-id", "commit-2") + .await + .expect("commit lookup should succeed") + .is_none() + ); + assert!( + stale_store + .get_commit_by_idempotency_key(bucket, "table-id", "client-request-2") + .await + .expect("idempotency lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn strong_catalog_backing_refreshes_hydrated_reads_after_independent_commit() { + let backend = TestCatalogObjectBackend::default(); + let writer = StrongTableCatalogStore::new(backend.clone()); + let reader = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + writer + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + writer + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + writer + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + let loaded_before_commit = reader + .load_table(bucket, "sales", "orders") + .await + .expect("reader should hydrate") + .expect("table should exist"); + assert_eq!(loaded_before_commit.metadata_location, current_metadata); + + let result = writer + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .expect("writer commit should succeed"); + + let loaded_after_commit = reader + .load_table(bucket, "sales", "orders") + .await + .expect("reader should refresh durable state") + .expect("table should still exist"); + assert_eq!(loaded_after_commit.metadata_location, result.table.metadata_location); + assert_eq!(loaded_after_commit.version_token, result.table.version_token); +} + +#[tokio::test] +async fn strong_catalog_backing_skips_snapshot_body_when_etag_is_unchanged() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + + backend.reset_call_counts().await; + let loaded = store + .get_table_bucket(bucket) + .await + .expect("table bucket load should succeed") + .expect("table bucket should exist"); + + assert_eq!(loaded.table_bucket, bucket); + assert_eq!(backend.metadata_call_count().await, 1); + assert_eq!(backend.read_call_count().await, 0); + assert_eq!(backend.list_call_count().await, 0); +} + +#[tokio::test] +async fn strong_catalog_backing_resolves_data_plane_resource_without_catalog_scan() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata)) + .await + .expect("table should be created"); + + backend.reset_call_counts().await; + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/file.parquet") + .await + .expect("data-plane lookup should succeed") + .expect("data-plane resource should resolve"); + + assert_eq!(resource.table_bucket, bucket); + assert_eq!(resource.namespace, namespace.public_name()); + assert_eq!(resource.table, table.as_str()); + assert_eq!(resource.table_id, "table-id"); + assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); + assert_eq!(backend.metadata_call_count().await, 1); + assert_eq!(backend.read_call_count().await, 0); + assert_eq!(backend.list_call_count().await, 0); +} + +#[tokio::test] +async fn strong_catalog_backing_does_not_publish_warehouse_index_before_snapshot_persist() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + + let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let create_store = store.clone(); + let create_task = tokio::spawn(async move { + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + create_store + .create_table(test_table_entry("analytics", &namespace, &table, current_metadata)) + .await + }); + pause.wait_started().await; + + let resource_before_commit = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/file.parquet") + .await + .expect("data-plane lookup should succeed while snapshot persist is paused"); + assert!(resource_before_commit.is_none()); + assert!( + store + .load_table(bucket, "sales", "orders") + .await + .expect("table load should succeed while snapshot persist is paused") + .is_none() + ); + + pause.release(); + create_task + .await + .expect("create task should join") + .expect("table should be created"); + + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/file.parquet") + .await + .expect("data-plane lookup should succeed after snapshot persist") + .expect("new table warehouse index should be visible after commit"); + assert_eq!(resource.table_bucket, bucket); + assert_eq!(resource.namespace, namespace.public_name()); + assert_eq!(resource.table, table.as_str()); + assert_eq!(resource.table_id, "table-id"); + assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); +} + +#[tokio::test] +async fn strong_catalog_backing_rejects_duplicate_snapshot_warehouse_index_entries() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let orders = IdentifierSegment::parse("orders").expect("table should parse"); + let returns = IdentifierSegment::parse("returns").expect("table should parse"); + let mut returns_entry = test_table_entry( + bucket, + &namespace, + &returns, + default_table_metadata_file_path(&namespace, &returns, "00001.metadata.json"), + ); + returns_entry.table_id = "table-id-2".to_string(); + returns_entry.table_uuid = "table-uuid-2".to_string(); + + let snapshot = StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: vec![test_namespace_entry(bucket, &namespace)], + tables: vec![ + test_table_entry( + bucket, + &namespace, + &orders, + default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), + ), + returns_entry, + ], + views: Vec::new(), + commits: Vec::new(), + idempotency: Vec::new(), + }; + backend + .seed_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + serde_json::to_vec(&snapshot).expect("strong snapshot should encode"), + ) + .await; + + let err = store + .get_table_bucket(bucket) + .await + .expect_err("duplicate warehouse prefix should fail snapshot hydration"); + + assert_matches!(err, TableCatalogStoreError::Invalid(message) if message.contains("duplicate active table warehouse location")); +} + +#[tokio::test] +async fn strong_catalog_backing_does_not_publish_draft_when_persist_and_reload_fail() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + backend.fail_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + backend.fail_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + + let err = store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect_err("namespace write should fail"); + + assert_matches!(err, TableCatalogStoreError::Internal(_)); + let state = store.state.lock().await; + assert!( + !state + .namespaces + .contains_key(&StrongTableCatalogStore::::namespace_key(bucket, &namespace)) + ); + drop(state); + + assert!( + store + .list_namespaces(bucket) + .await + .expect("durable state should reload after transient failure") + .is_empty() + ); +} + +#[tokio::test] +async fn strong_catalog_backing_commit_conflict_keeps_pointer_and_wal_unchanged() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + let err = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "append".to_string(), + expected_version_token: "stale-token".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Conflict(_)); + let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(loaded.metadata_location, current_metadata); + assert_eq!(loaded.version_token, "token-v1"); + assert!( + store + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_commit_by_idempotency_key(bucket, "table-id", "client-request") + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn strong_catalog_backing_rejects_invalid_table_warehouse_location() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + let mut entry = test_table_entry(bucket, &namespace, &table, current_metadata); + entry.warehouse_location = "s3://other-bucket/tables/table-id".to_string(); + + let err = store.create_table(entry).await.unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Invalid(_)); + assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_none()); +} + +#[tokio::test] +async fn strong_catalog_backing_rejects_duplicate_table_warehouse_location_on_register() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let orders = IdentifierSegment::parse("orders").unwrap(); + let customers = IdentifierSegment::parse("customers").unwrap(); + let orders_metadata = default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"); + let customers_metadata = default_table_metadata_file_path(&namespace, &customers, "00001.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &orders, orders_metadata)) + .await + .unwrap(); + let mut duplicate = test_table_entry(bucket, &namespace, &customers, customers_metadata); + duplicate.table_id = "table-id-2".to_string(); + duplicate.table_uuid = "table-uuid-2".to_string(); + duplicate.warehouse_location = format!("s3://{bucket}/tables/table-id"); + + let err = store.register_table(duplicate).await.unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Conflict(_)); + assert!(store.load_table(bucket, "sales", "customers").await.unwrap().is_none()); +} + +#[tokio::test] +async fn strong_catalog_backing_rejects_duplicate_table_warehouse_location_on_commit_relocation() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let orders = IdentifierSegment::parse("orders").unwrap(); + let customers = IdentifierSegment::parse("customers").unwrap(); + let orders_metadata = default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"); + let relocated_metadata = default_table_metadata_file_path(&namespace, &orders, "00002.metadata.json"); + let customers_metadata = default_table_metadata_file_path(&namespace, &customers, "00001.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &orders, orders_metadata.clone())) + .await + .unwrap(); + let mut customers_entry = test_table_entry(bucket, &namespace, &customers, customers_metadata); + customers_entry.table_id = "table-id-2".to_string(); + customers_entry.table_uuid = "table-uuid-2".to_string(); + customers_entry.warehouse_location = format!("s3://{bucket}/tables/customer-id"); + store.create_table(customers_entry).await.unwrap(); + backend + .seed_object( + bucket, + &relocated_metadata, + serde_json::to_vec(&serde_json::json!({ + "location": "s3://analytics/tables/customer-id", + "table-uuid": "table-uuid" + })) + .unwrap(), + ) + .await; + + let err = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: orders.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "set-location".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: orders_metadata.clone(), + new_metadata_location: relocated_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Conflict(_)); + let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(loaded.metadata_location, orders_metadata); + assert_eq!(loaded.warehouse_location, format!("s3://{bucket}/tables/table-id")); + assert!( + store + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn diagnostics_backing_manifest_requires_recovery_before_migration() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; + + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap(); + + let diagnostics = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); + + assert_eq!(diagnostics.backing_manifest.current.wal.finalization_required_count, 1); + assert_eq!( + diagnostics.backing_manifest.migration.status, + TableCatalogBackingMigrationStatus::RecoveryRequired + ); + assert!( + diagnostics + .backing_manifest + .migration + .blockers + .contains(&TableCatalogBackingMigrationBlocker::CommitRecoveryRequired) + ); +} + +#[tokio::test] +async fn consistency_check_reports_missing_metadata_object() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + + let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); + + assert_eq!(report.catalog.table.metadata_location, current.clone()); + assert_eq!(report.current_metadata_status, TableMetadataPointerStatus::MissingObject); + assert_eq!(report.recovery_status, TableCatalogRecoveryStatus::ReadOnlyRecommended); + assert_eq!(report.recommended_actions, vec![TableCatalogRecoveryAction::RestoreCurrentMetadataObject]); + assert!(report.orphan_metadata_candidate_locations.is_empty()); +} + +#[tokio::test] +async fn consistency_check_reports_invalid_metadata_location() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let invalid_metadata = ".rustfs-table/warehouses/default/namespaces/sales/tables/other/metadata/00001.metadata.json"; + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, invalid_metadata.to_string())) + .await + .unwrap(); + + let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); + + assert_eq!(report.catalog.table.metadata_location, invalid_metadata); + assert_eq!(report.current_metadata_status, TableMetadataPointerStatus::InvalidLocation); + assert!(report.orphan_metadata_candidate_locations.is_empty()); +} + +#[tokio::test] +async fn orphan_metadata_scan_does_not_treat_largest_version_as_committed() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let uncommitted = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + backend + .seed_object(bucket, &uncommitted, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); + + assert_eq!(report.current_metadata_status, TableMetadataPointerStatus::Valid); + assert_eq!(report.catalog.table.metadata_location, current); + assert_eq!(report.orphan_metadata_candidate_locations, vec![uncommitted]); +} + +#[tokio::test] +async fn orphan_metadata_scan_keeps_metadata_for_protected_snapshot_refs() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let orphan = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let tagged = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let current = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend.seed_object(bucket, &orphan, b"{}".to_vec()).await; + backend + .seed_object( + bucket, + &tagged, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 10 + })) + .unwrap(), + ) + .await; + backend + .seed_object( + bucket, + ¤t, + serde_json::to_vec(&serde_json::json!({ + "current-snapshot-id": 20, + "metadata-log": [], + "refs": { + "audit": { + "snapshot-id": 10, + "type": "tag" + } + } + })) + .unwrap(), + ) + .await; + + let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); + + assert_eq!(report.current_metadata_status, TableMetadataPointerStatus::Valid); + assert_eq!(report.orphan_metadata_candidate_locations, vec![orphan]); +} + +#[tokio::test] +async fn object_table_catalog_store_commits_with_token_match_and_writes_log() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + let result = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client/%2f\nrequest".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata.clone(), + requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap(); + + assert_eq!(result.table.metadata_location, new_metadata); + assert_ne!(result.table.version_token, "token-v1"); + assert_eq!(result.table.generation, 2); + assert_eq!(result.commit_log.status, CommitLogStatus::Committed); + + let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(loaded.metadata_location, result.table.metadata_location); + assert_eq!(loaded.version_token, result.table.version_token); + assert!( + store + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .unwrap() + .is_some() + ); + assert!( + store + .get_commit_by_idempotency_key(bucket, "table-id", "client/%2f\nrequest") + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn object_table_catalog_store_syncs_warehouse_location_from_committed_metadata() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + store.backfill_table_warehouse_index(bucket).await.unwrap(); + backend.reset_call_counts().await; + backend + .seed_object( + bucket, + &new_metadata, + serde_json::to_vec(&serde_json::json!({ + "location": "s3://analytics/tables/relocated-table-id", + "table-uuid": "table-uuid" + })) + .unwrap(), + ) + .await; + + let result = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "set-location".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap(); + + assert_eq!(result.table.warehouse_location, "s3://analytics/tables/relocated-table-id"); + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/relocated-table-id/data/part-00001.parquet") + .await + .expect("data-plane resource lookup should succeed") + .expect("relocated table warehouse object should resolve to the table"); + assert_eq!(resource.table, "orders"); + assert_eq!(resource.warehouse_object_prefix, "tables/relocated-table-id/"); + let old_resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("old table warehouse lookup should succeed"); + assert!(old_resource.is_none()); + assert_eq!(backend.list_call_count().await, 0); +} + +#[tokio::test] +async fn object_table_catalog_store_reuses_old_prefix_after_failed_relocation_index_delete() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let next_table = IdentifierSegment::parse("returns").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let old_index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + let old_entry = test_table_entry(bucket, &namespace, &table, current_metadata.clone()); + store + .create_table(old_entry.clone()) + .await + .expect("old table should be created"); + backend + .seed_object( + bucket, + &new_metadata, + serde_json::to_vec(&serde_json::json!({ + "location": "s3://analytics/tables/relocated-table-id", + "table-uuid": "table-uuid" + })) + .unwrap(), + ) + .await; + backend.fail_delete_attempt(RUSTFS_META_BUCKET, &old_index_path, 1).await; + + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "set-location".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: new_metadata, + requirements: vec![serde_json::json!({"type": "assert-table-uuid", "uuid": "table-uuid"})], + writer: Some("pyiceberg/test".to_string()), + }) + .await + .unwrap(); + + let mut next_entry = test_table_entry(bucket, &namespace, &next_table, current_metadata); + next_entry.table_id = "next-table-id".to_string(); + next_entry.warehouse_location = format!("s3://{bucket}/tables/table-id"); + store + .create_table(next_entry) + .await + .expect("stale old warehouse index should not block prefix reuse"); + + let (index, _) = store + .read_entry::(store.catalog_bucket(), &old_index_path) + .await + .unwrap() + .expect("reused prefix index should exist"); + assert_eq!(index.table, "returns"); + assert_eq!(index.table_id, "next-table-id"); + + store + .delete_table_warehouse_index(&old_entry) + .await + .expect("old owner should not delete reused prefix index"); + let reused_resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("reused prefix lookup should succeed") + .expect("reused prefix should still resolve to the new table"); + assert_eq!(reused_resource.table, "returns"); + assert_eq!(reused_resource.table_id, "next-table-id"); +} + +#[tokio::test] +async fn object_table_catalog_store_does_not_advance_table_when_idempotency_staging_fails() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let idempotency_key = "client-request"; + let idempotency_path = TableCatalogObjectPaths::default().commit_idempotency_entry_path(bucket, "table-id", idempotency_key); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &idempotency_path, 1).await; + + let err = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some(idempotency_key.to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Internal(_)); + let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(loaded.metadata_location, current_metadata); + assert_eq!(loaded.version_token, "token-v1"); + let staged = store.get_commit_by_id(bucket, "table-id", "commit-1").await.unwrap().unwrap(); + assert_eq!(staged.status, CommitLogStatus::Staged); +} + +#[tokio::test] +async fn object_table_catalog_store_recovers_staged_commit_after_post_cas_finalization_failure() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; + + let request = TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata.clone(), + requirements: Vec::new(), + writer: None, + }; + + let result = store.commit_table(request.clone()).await.unwrap(); + + assert_eq!(result.table.metadata_location, new_metadata); + assert_eq!(result.commit_log.status, CommitLogStatus::Committed); + let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(loaded.version_token, result.table.version_token); + let staged = store.get_commit_by_id(bucket, "table-id", "commit-1").await.unwrap().unwrap(); + assert_eq!(staged.status, CommitLogStatus::Staged); + + let retry = store.commit_table(request).await.unwrap(); + assert_eq!(retry.table.version_token, result.table.version_token); + assert_eq!(retry.commit_log.status, CommitLogStatus::Committed); + let committed = store.get_commit_by_id(bucket, "table-id", "commit-1").await.unwrap().unwrap(); + assert_eq!(committed.status, CommitLogStatus::Committed); +} + +#[tokio::test] +async fn table_commit_recovery_reports_post_cas_staged_commit() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; + + let result = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata.clone(), + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap(); + assert_eq!(result.commit_log.status, CommitLogStatus::Committed); + + let report = store.plan_table_commit_recovery(bucket, "sales", "orders").await.unwrap(); + + assert_eq!(report.current_metadata_location, new_metadata); + assert_eq!(report.finalization_required_count, 1); + assert_eq!(report.manual_review_count, 0); + assert_eq!(report.commits.len(), 1); + assert_eq!(report.commits[0].commit_id, "commit-1"); + assert_eq!(report.commits[0].status, CommitLogStatus::Staged); + assert_eq!(report.commits[0].recovery_state, TableCommitRecoveryState::FinalizationRequired); +} + +#[tokio::test] +async fn diagnostics_report_includes_table_commit_recovery_state() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; + + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap(); + + let report = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); + + assert_eq!(report.recovery_status, TableCatalogRecoveryStatus::Recoverable); + assert_eq!(report.recommended_actions, vec![TableCatalogRecoveryAction::RunCommitRecovery]); + assert_eq!(report.commit_recovery.finalization_required_count, 1); + assert_eq!(report.commit_recovery.commits.len(), 1); + assert_eq!( + report.commit_recovery.commits[0].recovery_state, + TableCommitRecoveryState::FinalizationRequired + ); +} + +#[tokio::test] +async fn table_commit_recovery_finalizes_post_cas_staged_commit() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &commit_path, 2).await; + + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap(); + + let report = store.recover_table_commits(bucket, "sales", "orders").await.unwrap(); + + assert_eq!(report.finalized_count, 1); + assert_eq!(report.finalization_required_count, 0); + let committed = store.get_commit_by_id(bucket, "table-id", "commit-1").await.unwrap().unwrap(); + assert_eq!(committed.status, CommitLogStatus::Committed); +} + +#[tokio::test] +async fn table_commit_recovery_reports_staged_commit_after_table_cas_failure() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let table_path = TableCatalogObjectPaths::default().table_entry_path(bucket, &namespace, &table); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, ¤t_metadata, b"{}".to_vec()).await; + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &table_path, 2).await; + + let err = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap_err(); + assert_matches!(err, TableCatalogStoreError::Internal(_)); + let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(loaded.metadata_location, current_metadata); + + let report = store.plan_table_commit_recovery(bucket, "sales", "orders").await.unwrap(); + assert_eq!(report.staged_before_table_update_count, 1); + assert_eq!(report.finalization_required_count, 0); + assert_eq!(report.commits[0].recovery_state, TableCommitRecoveryState::StagedBeforeTableUpdate); + + let diagnostics = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap(); + assert_eq!(diagnostics.current_metadata_status, TableMetadataPointerStatus::Valid); + assert_eq!(diagnostics.recovery_status, TableCatalogRecoveryStatus::Recoverable); + assert_eq!(diagnostics.recommended_actions, vec![TableCatalogRecoveryAction::RetryCommit]); +} + +#[tokio::test] +async fn table_commit_recovery_repairs_stale_idempotency_index_after_partial_finalization() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let idempotency_path = + TableCatalogObjectPaths::default().commit_idempotency_entry_path(bucket, "table-id", "client-request-1"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &idempotency_path, 2).await; + + let result = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap(); + assert_eq!(result.commit_log.status, CommitLogStatus::Committed); + let stale_index = store + .get_commit_by_idempotency_key(bucket, "table-id", "client-request-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stale_index.status, CommitLogStatus::Staged); + + let report = store.plan_table_commit_recovery(bucket, "sales", "orders").await.unwrap(); + assert_eq!(report.idempotency_repair_required_count, 1); + assert_eq!(report.manual_review_count, 0); + assert_eq!(report.commits[0].recovery_state, TableCommitRecoveryState::IdempotencyIndexRepairRequired); + assert_eq!(report.commits[0].idempotency_index_status, TableCommitIdempotencyIndexStatus::Stale); + + let repaired = store.recover_table_commits(bucket, "sales", "orders").await.unwrap(); + + assert_eq!(repaired.finalized_count, 1); + assert_eq!(repaired.idempotency_repair_required_count, 0); + let repaired_index = store + .get_commit_by_idempotency_key(bucket, "table-id", "client-request-1") + .await + .unwrap() + .unwrap(); + assert_eq!(repaired_index.status, CommitLogStatus::Committed); +} + +#[tokio::test] +async fn object_table_catalog_store_rejects_stale_commit_token() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .unwrap(); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + let err = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "stale-token".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .unwrap_err(); + + assert_matches!(err, TableCatalogStoreError::Conflict(_)); + let loaded = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap(); + assert_eq!(loaded.metadata_location, current_metadata); + assert_eq!(loaded.version_token, "token-v1"); + assert!( + store + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .unwrap() + .is_none() + ); +} + +#[test] +fn namespace_marker_path_stays_under_default_reserved_boundary() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + + assert_eq!( + default_namespace_marker_path(&namespace), + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/namespace.json" + ); + assert_eq!(namespace.public_name(), "analytics.daily_events"); +} + +#[test] +fn namespace_marker_path_extracts_public_name() { + assert_eq!( + namespace_name_from_marker_path(".rustfs-table/warehouses/default/namespaces/analytics/daily_events/namespace.json"), + Some("analytics.daily_events".to_string()) + ); + assert_eq!( + namespace_name_from_marker_path( + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/current.json" + ), + None + ); + assert_eq!( + namespace_name_from_marker_path(".rustfs-table/warehouses/other/namespaces/analytics/daily_events/namespace.json"), + None + ); +} + +#[test] +fn namespace_marker_json_uses_stable_catalog_defaults() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + let marker = serde_json::to_value(NamespaceMarker::new(&namespace)).unwrap(); + + assert_eq!(marker["version"], TABLE_NAMESPACE_MARKER_VERSION); + assert_eq!(marker["namespace"], "analytics.daily_events"); + assert!(!namespace_marker_json(&namespace).unwrap().is_empty()); +} + +#[test] +fn table_marker_path_stays_under_namespace_reserved_boundary() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + let table = IdentifierSegment::parse("events").unwrap(); + + assert_eq!( + default_table_root_prefix(&namespace), + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/" + ); + assert_eq!( + default_table_marker_path(&namespace, &table), + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/table.json" + ); +} + +#[test] +fn table_marker_path_extracts_table_name() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + + assert_eq!( + table_name_from_marker_path( + &namespace, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/table.json" + ), + Some("events".to_string()) + ); + assert_eq!( + table_name_from_marker_path( + &namespace, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/current.json" + ), + None + ); + assert_eq!( + table_name_from_marker_path( + &namespace, + ".rustfs-table/warehouses/default/namespaces/analytics/other/tables/events/table.json" + ), + None + ); +} + +#[test] +fn table_marker_json_uses_stable_catalog_defaults() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + let table = IdentifierSegment::parse("events").unwrap(); + let marker = serde_json::to_value(TableMarker::new(&namespace, &table)).unwrap(); + + assert_eq!(marker["version"], TABLE_RESOURCE_MARKER_VERSION); + assert_eq!(marker["namespace"], "analytics.daily_events"); + assert_eq!(marker["name"], "events"); + assert!(marker["metadata_location"].is_null()); + assert!(!table_marker_json(&namespace, &table).unwrap().is_empty()); +} + +#[test] +fn table_current_pointer_path_stays_under_table_boundary() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + let table = IdentifierSegment::parse("events").unwrap(); + + assert_eq!( + default_table_metadata_dir_path(&namespace, &table), + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata" + ); + assert_eq!( + default_table_current_pointer_path(&namespace, &table), + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/current.json" + ); + assert_eq!( + default_table_lifecycle_path(&namespace, &table), + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/lifecycle.json" + ); +} + +#[test] +fn table_metadata_file_path_stays_under_metadata_boundary() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + let table = IdentifierSegment::parse("events").unwrap(); + let table_identifier = + TableIdentifier::new(IdentifierSegment::parse(DEFAULT_WAREHOUSE_ID).unwrap(), namespace.clone(), table.clone()); + + assert_eq!( + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" + ); + assert_eq!( + TablePathResolver::default().metadata_file_path(&table_identifier, "00001.metadata.json"), + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" + ); +} + +#[test] +fn table_metadata_file_name_validation_rejects_unsafe_names() { + assert!(is_valid_table_metadata_file_name("00001.metadata.json")); + assert!(is_valid_table_metadata_file_name("v1-4f2c_metadata.json")); + + assert!(!is_valid_table_metadata_file_name("")); + assert!(!is_valid_table_metadata_file_name(".metadata.json")); + assert!(!is_valid_table_metadata_file_name("00001.metadata")); + assert!(!is_valid_table_metadata_file_name("00001.JSON")); + assert!(!is_valid_table_metadata_file_name("../current.json")); + assert!(!is_valid_table_metadata_file_name("nested/00001.json")); + assert!(!is_valid_table_metadata_file_name("nested%2f00001.json")); + assert!(!is_valid_table_metadata_file_name("00001\\metadata.json")); + assert!(!is_valid_table_metadata_file_name("00001\nmetadata.json")); +} + +#[test] +fn table_metadata_location_validation_stays_inside_metadata_dir() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + let table = IdentifierSegment::parse("events").unwrap(); + + assert!(is_valid_table_metadata_location( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" + )); + assert!(!is_valid_table_metadata_location(&namespace, &table, "")); + assert!(!is_valid_table_metadata_location( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/current.json" + )); + assert!(!is_valid_table_metadata_location( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/other/metadata/00001.json" + )); + assert!(!is_valid_table_metadata_location( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/../current.json" + )); + assert!(!is_valid_table_metadata_location( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/nested/00001.json" + )); +} + +#[test] +fn metadata_location_from_metadata_file_path_extracts_table_metadata_only() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + let table = IdentifierSegment::parse("events").unwrap(); + + assert_eq!( + metadata_location_from_metadata_file_path( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" + ), + Some( + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.metadata.json" + .to_string() + ) + ); + assert_eq!( + metadata_location_from_metadata_file_path( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/current.json" + ), + None + ); + assert_eq!( + metadata_location_from_metadata_file_path( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/nested/00001.metadata.json" + ), + None + ); + assert_eq!( + metadata_location_from_metadata_file_path( + &namespace, + &table, + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/other/metadata/00001.metadata.json" + ), + None + ); +} + +#[test] +fn table_metadata_pointer_json_round_trips() { + let location = + ".rustfs-table/warehouses/default/namespaces/analytics/daily_events/tables/events/metadata/00001.json".to_string(); + let data = table_metadata_pointer_json(location.clone()).unwrap(); + let pointer = parse_table_metadata_pointer(&data).unwrap(); + + assert_eq!(pointer.version, TABLE_METADATA_POINTER_VERSION); + assert_eq!(pointer.metadata_location, location); +} + +#[test] +fn object_mutation_entrypoints_call_reserved_prefix_guard() { + let source = include_str!("../app/object_usecase.rs"); + let delete_object = source + .split_once("pub async fn execute_delete_object") + .and_then(|(_, remainder)| remainder.split_once("pub async fn execute_head_object")) + .map(|(delete_object, _)| delete_object) + .expect("delete object entrypoint should remain in the object usecase"); + + for expected in [ + "validate_object_key(&key, request_method_name)?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", + "validate_object_key(&key, \"COPY (dest)\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", + "if let Err(err) = validate_table_catalog_object_mutation(&bucket, &obj_id.key).await", + "validate_table_catalog_object_mutation(&bucket, &object).await?;", + "validate_object_key(&key, \"PUT\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", + "validate_table_catalog_object_mutation(&bucket, &fpath).await?;", + ] { + assert!(source.contains(expected), "missing object mutation guard: {expected}"); + } + assert!( + delete_object.contains("validate_object_key(&key, \"DELETE\")?;") + && delete_object.contains("validate_table_catalog_object_mutation(&bucket, &key).await?;"), + "delete object entrypoint must validate the object key and reserved catalog prefix" + ); +} + +#[test] +fn multipart_mutation_entrypoints_call_reserved_prefix_guard() { + let source = include_str!("../app/multipart_usecase.rs"); + + assert_eq!( + source + .matches("validate_table_catalog_object_mutation(&bucket, &key).await?;") + .count(), + 4 + ); +} + +#[test] +fn object_metadata_mutation_entrypoints_call_reserved_prefix_guard() { + let source = include_str!("../storage/ecfs.rs"); + + assert_eq!( + source + .matches("validate_table_catalog_object_mutation(&bucket, &object).await?;") + .count(), + 2 + ); + assert_eq!( + source + .matches("validate_table_catalog_object_mutation(&bucket, &key).await?;") + .count(), + 2 + ); +} + +#[test] +fn identifier_segment_accepts_conservative_catalog_names() { + for value in [ + "a", + "a1", + "a-b", + "a_b", + "abc123", + "a23456789012345678901234567890123456789012345678901234567890123", + ] { + assert_eq!(IdentifierSegment::parse(value).unwrap().as_str(), value); + } +} + +#[test] +fn identifier_segment_rejects_ambiguous_or_unsafe_names() { + for value in [ + "", + ".", + "..", + "Upper", + "has.dot", + "has/slash", + "has\\slash", + "has%2fslash", + "-leading", + "trailing-", + "_leading", + "trailing_", + "has space", + "name\nbreak", + ] { + assert!(IdentifierSegment::parse(value).is_err(), "value should be rejected: {value:?}"); + } + + let too_long = "a".repeat(IdentifierSegment::MAX_LEN + 1); + assert!(IdentifierSegment::parse(too_long).is_err()); +} + +#[test] +fn namespace_uses_dot_syntax_for_public_identity_and_slash_for_storage() { + let namespace = Namespace::parse("analytics.daily_events").unwrap(); + + assert_eq!(namespace.segments().len(), 2); + assert_eq!(namespace.storage_id(), "analytics/daily_events"); +} + +#[test] +fn namespace_length_is_bounded_for_catalog_paths_and_page_tokens() { + let mut segments = vec!["a".repeat(63); 8]; + segments[0].push('a'); + let max_length_namespace = segments.join("."); + assert_eq!(max_length_namespace.len(), Namespace::MAX_LEN); + Namespace::parse(&max_length_namespace).expect("namespace at the maximum length should parse"); + + let namespace = format!("{max_length_namespace}.a"); + assert_eq!( + Namespace::parse(&namespace), + Err(CatalogIdentifierError::NamespaceTooLong { max: Namespace::MAX_LEN }) + ); +} + +#[test] +fn resolver_builds_paths_under_reserved_table_boundary() { + let table = TableIdentifier::new( + IdentifierSegment::parse("warehouse1").unwrap(), + Namespace::parse("analytics.daily").unwrap(), + IdentifierSegment::parse("events").unwrap(), + ); + let resolver = TablePathResolver::default(); + + assert_eq!( + resolver.current_pointer_path(&table), + ".rustfs-table/warehouses/warehouse1/namespaces/analytics/daily/tables/events/current.json" + ); + assert_eq!( + resolver.metadata_dir_path(&table), + ".rustfs-table/warehouses/warehouse1/namespaces/analytics/daily/tables/events/metadata" + ); +}