mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 16:46:55 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bae602a5ef | |||
| d9071b1b83 | |||
| e3c15f012c |
@@ -182,12 +182,7 @@ jobs:
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Readers: test-and-lint-rio-v2 (per-PR), build-rustfs-debug-binary-rio-v2
|
||||
# (weekly schedule / manual dispatch only — dormant rio-v2 variant, see
|
||||
# rustfs/backlog#1835 and docs/architecture/minio-file-format-compat.md).
|
||||
# The second build below stays despite the reduced cadence: it warms the
|
||||
# rio-v2,e2e-test-hooks feature resolution the scheduled build restores,
|
||||
# which keeps that lane inside its 30-minute timeout.
|
||||
# Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
|
||||
warm-ci-feat-rio:
|
||||
name: Warm ci-feat-rio
|
||||
runs-on: sm-standard-4
|
||||
|
||||
@@ -533,12 +533,7 @@ jobs:
|
||||
|
||||
build-rustfs-debug-binary-rio-v2:
|
||||
name: Build RustFS Debug Binary (rio-v2)
|
||||
# Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
|
||||
# default build, so this full-suite lane runs only on the weekly schedule
|
||||
# and manual dispatch. Per-PR cfg-seam coverage stays with
|
||||
# test-and-lint-rio-v2. Lifecycle and the promote-or-delete condition:
|
||||
# docs/architecture/minio-file-format-compat.md ("rio-v2 variant lifecycle").
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
@@ -829,9 +824,6 @@ jobs:
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
name: End-to-End Tests (rio-v2)
|
||||
# Inherits the schedule/dispatch-only gate through needs: on every other
|
||||
# event build-rustfs-debug-binary-rio-v2 is skipped, so this job skips
|
||||
# with it (see the dormant-variant comment on that job).
|
||||
needs: [ build-rustfs-debug-binary-rio-v2 ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
|
||||
+1
-4
@@ -101,10 +101,7 @@ refactors.
|
||||
|
||||
The `rustfs` binary crate composes these libraries into the running server.
|
||||
`ecstore` remains the storage engine at the architectural center; its internal
|
||||
module split is tracked under `docs/architecture/`. `rio-v2` is the
|
||||
feature-gated MinIO on-disk format compatibility I/O layer; it ships in no
|
||||
default build (lifecycle:
|
||||
[docs/architecture/minio-file-format-compat.md](docs/architecture/minio-file-format-compat.md)).
|
||||
module split is tracked under `docs/architecture/`.
|
||||
|
||||
## Architecture Invariants
|
||||
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ members = [
|
||||
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build)
|
||||
"crates/rio-v2", # Next-generation Rust I/O compatibility layer
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
|
||||
@@ -34,7 +34,6 @@ for later deletion.
|
||||
- `tonic-013-status-render` peer RPC failure classification: internode failures that reach a node only as text (a peer's error_info payload, a status flattened through format!) are classified by matching the rendering of an Unavailable gRPC status. Releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered that status as "status: Unavailable, message: ..."; tonic 0.14 renders it as "code: 'The service is currently unavailable', message: ...". Both forms are matched so an older peer's relayed text still marks an unreachable peer offline. Remove the tonic 0.13 form after the minimum supported RustFS peer version ships tonic 0.14 or later.
|
||||
- `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection.
|
||||
- `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object.
|
||||
- `rio-v2-dormant-variant` dormant `rio-v2` build variant: `crates/rio-v2` and the `rio-v2` feature ship in no default or release build and exist only as the candidate MinIO stream-format implementation for the rustfs/backlog#1638 SSE-interop adjudication. Per-PR CI keeps only `test-and-lint-rio-v2` to guard the `#[cfg(feature = "rio-v2")]` seam; the full-suite lanes (`build-rustfs-debug-binary-rio-v2`, `e2e-tests-rio-v2` in `.github/workflows/ci.yml`) run on schedule/workflow_dispatch only — see the "`rio-v2` variant lifecycle" section in [minio-file-format-compat.md](minio-file-format-compat.md). While both implementations exist, DARE/S2 stream fixes must land in both `crates/rio` and `crates/rio-v2`. No `RUSTFS_COMPAT_TODO` source marker applies: the temporary surface is CI workflow YAML plus an entire feature-gated candidate crate, not a compatibility code path inside shipping code, and workflow files are outside the marker convention's Rust scope. Remove after the #1638 adjudication lands and converges on one implementation: delete the losing implementation, its feature seam, and the gating CI jobs.
|
||||
|
||||
## Review Checklist
|
||||
|
||||
|
||||
@@ -271,32 +271,6 @@ Seam 2 surfaces its own error, but only for objects that got past seam 1.
|
||||
|
||||
The interop harness reflects this. The reader tests are `#[ignore]` (`rustfs/src/storage/minio_generated_read_test.rs:244`, `:250`), the workflow that would run them is disabled at the GitHub Actions level and states in its own header that end-to-end MinIO-to-RustFS SSE interop is not implemented (`.github/workflows/minio-interop.yml:24-29`, `:34-39`), and the fixture suite's scope note says the tests "do not yet validate full plaintext reconstruction from MinIO-written encrypted data" (`crates/rio-v2/tests/README.md:55`).
|
||||
|
||||
### `rio-v2` variant lifecycle
|
||||
|
||||
The variant is deliberately **dormant** until rustfs/backlog#1638 is
|
||||
adjudicated. Dormant means:
|
||||
|
||||
- **Per-PR CI keeps one guard job.** Only `test-and-lint-rio-v2` in
|
||||
`.github/workflows/ci.yml` runs per PR; its job is to keep the
|
||||
`#[cfg(feature = "rio-v2")]` seam compiling and its unit tests green so the
|
||||
variant does not bit-rot. The full-suite lanes —
|
||||
`build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` — run only on the
|
||||
weekly `schedule` and on `workflow_dispatch`, not per PR, per main push, or
|
||||
in the merge queue.
|
||||
- **Post-1.0 the variant is promoted or deleted.** The #1638 adjudication
|
||||
converges on one implementation: either `rio-v2` becomes a shipped
|
||||
configuration, or the losing side is removed together with its feature seam
|
||||
and its gating CI jobs. Tracked as `rio-v2-dormant-variant` in
|
||||
[compat-cleanup-register.md](compat-cleanup-register.md).
|
||||
- **DARE/S2 fixes land in both crates.** While both implementations exist,
|
||||
any fix to the DARE V2 stream format or the S2 compression framing/index
|
||||
must be applied to `crates/rio` **and** `crates/rio-v2` (each has its own
|
||||
`encrypt_reader.rs` and `compress_reader.rs`). Both implement the same
|
||||
stream primitives; a single-sided fix forks on-disk behavior between
|
||||
default and `rio-v2` builds and invalidates the dormant variant as an
|
||||
interop baseline — and with full-suite CI now weekly-only, the divergence
|
||||
could go unnoticed for up to a week.
|
||||
|
||||
### Reverse direction
|
||||
|
||||
Migrating back is also unsupported. Under `rio-v2` RustFS writes its own DEK envelope into MinIO's sealed-key metadata slots and labels it with MinIO's seal algorithm (`rustfs/src/storage/sse.rs:1830-1852`), so the metadata is MinIO-shaped while the key bytes are not MinIO-openable. Default builds do not populate those slots at all (`rustfs/src/storage/sse.rs:1796-1798`). Treat RustFS-written SSE objects as readable only by RustFS.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -361,5 +361,8 @@ fn storage_error_to_catalog(action: &str, err: StorageError) -> TableCatalogStor
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,941 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared table-catalog test fixtures (backlog#1837).
|
||||
//!
|
||||
//! Pure data constructors for Iceberg metadata JSON and avro
|
||||
//! manifest-list/manifest bytes, shared by the store-level tests
|
||||
//! (`table_catalog/tests.rs`) and the admin handler tests
|
||||
//! (`admin/handlers/table_catalog/tests.rs`). The parameterized admin
|
||||
//! variants are canonical; the store tests wrap them with their historical
|
||||
//! fixed values (sequence 7 / snapshot 20), which keeps every produced byte
|
||||
//! identical to the pre-extraction fixtures.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::{
|
||||
StrongTableCatalogRuntime, TableCatalogObject, TableCatalogObjectBackend, TableCatalogObjectMetadata,
|
||||
TableCatalogPutPrecondition, TableCatalogStoreError, TableCatalogStoreResult, TableCommitPublication,
|
||||
};
|
||||
|
||||
pub(crate) fn table_metadata_json(table_uuid: &str, location: &str) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"format-version": 2,
|
||||
"table-uuid": table_uuid,
|
||||
"location": location,
|
||||
"last-sequence-number": 0,
|
||||
"last-updated-ms": 1,
|
||||
"last-column-id": 1,
|
||||
"schemas": [{
|
||||
"type": "struct",
|
||||
"schema-id": 0,
|
||||
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]
|
||||
}],
|
||||
"current-schema-id": 0,
|
||||
"partition-specs": [{"spec-id": 0, "fields": []}],
|
||||
"default-spec-id": 0,
|
||||
"last-partition-id": 999,
|
||||
"sort-orders": [{"order-id": 0, "fields": []}],
|
||||
"default-sort-order-id": 0,
|
||||
"properties": {},
|
||||
"snapshots": [],
|
||||
"snapshot-log": [],
|
||||
"metadata-log": [],
|
||||
"refs": {}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn manifest_list_avro_bytes(manifest_paths: &[&str], sequence_number: i64, snapshot_id: i64) -> Vec<u8> {
|
||||
let manifests = manifest_paths
|
||||
.iter()
|
||||
.map(|manifest_path| (*manifest_path, 0, sequence_number, snapshot_id))
|
||||
.collect::<Vec<_>>();
|
||||
manifest_list_avro_entries_with_partition_specs(&manifests)
|
||||
}
|
||||
|
||||
pub(crate) fn manifest_list_avro_entries(manifests: &[(&str, i64, i64)]) -> Vec<u8> {
|
||||
let manifests = manifests
|
||||
.iter()
|
||||
.map(|(manifest_path, sequence_number, snapshot_id)| (*manifest_path, 0, *sequence_number, *snapshot_id))
|
||||
.collect::<Vec<_>>();
|
||||
manifest_list_avro_entries_with_partition_specs(&manifests)
|
||||
}
|
||||
|
||||
pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str, i32, i64, i64)]) -> Vec<u8> {
|
||||
let schema = apache_avro::Schema::parse_str(
|
||||
r#"
|
||||
{
|
||||
"type": "record",
|
||||
"name": "manifest_file",
|
||||
"fields": [
|
||||
{"name": "manifest_path", "type": "string"},
|
||||
{"name": "manifest_length", "type": "long"},
|
||||
{"name": "partition_spec_id", "type": "int"},
|
||||
{"name": "content", "type": "int"},
|
||||
{"name": "sequence_number", "type": "long"},
|
||||
{"name": "min_sequence_number", "type": "long"},
|
||||
{"name": "added_snapshot_id", "type": "long"},
|
||||
{"name": "added_files_count", "type": "int"},
|
||||
{"name": "existing_files_count", "type": "int"},
|
||||
{"name": "deleted_files_count", "type": "int"},
|
||||
{"name": "added_rows_count", "type": "long"},
|
||||
{"name": "existing_rows_count", "type": "long"},
|
||||
{"name": "deleted_rows_count", "type": "long"}
|
||||
]
|
||||
}
|
||||
"#,
|
||||
)
|
||||
.expect("manifest list avro schema should parse");
|
||||
let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest list writer should initialize");
|
||||
for (manifest_path, partition_spec_id, sequence_number, snapshot_id) in manifests {
|
||||
writer
|
||||
.append_value(apache_avro::types::Value::Record(vec![
|
||||
(
|
||||
"manifest_path".to_string(),
|
||||
apache_avro::types::Value::String((*manifest_path).to_string()),
|
||||
),
|
||||
("manifest_length".to_string(), apache_avro::types::Value::Long(1)),
|
||||
("partition_spec_id".to_string(), apache_avro::types::Value::Int(*partition_spec_id)),
|
||||
("content".to_string(), apache_avro::types::Value::Int(0)),
|
||||
("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)),
|
||||
("min_sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)),
|
||||
("added_snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)),
|
||||
("added_files_count".to_string(), apache_avro::types::Value::Int(1)),
|
||||
("existing_files_count".to_string(), apache_avro::types::Value::Int(0)),
|
||||
("deleted_files_count".to_string(), apache_avro::types::Value::Int(0)),
|
||||
("added_rows_count".to_string(), apache_avro::types::Value::Long(1)),
|
||||
("existing_rows_count".to_string(), apache_avro::types::Value::Long(0)),
|
||||
("deleted_rows_count".to_string(), apache_avro::types::Value::Long(0)),
|
||||
]))
|
||||
.expect("manifest list record should append");
|
||||
}
|
||||
writer.into_inner().expect("manifest list avro bytes should flush")
|
||||
}
|
||||
|
||||
pub(crate) fn manifest_avro_bytes(files: &[(&str, i32, i32, i64, i64)]) -> Vec<u8> {
|
||||
let schema = apache_avro::Schema::parse_str(
|
||||
r#"
|
||||
{
|
||||
"type": "record",
|
||||
"name": "manifest_entry",
|
||||
"fields": [
|
||||
{"name": "status", "type": "int"},
|
||||
{"name": "snapshot_id", "type": "long"},
|
||||
{"name": "sequence_number", "type": "long"},
|
||||
{"name": "file_sequence_number", "type": "long"},
|
||||
{
|
||||
"name": "data_file",
|
||||
"type": {
|
||||
"type": "record",
|
||||
"name": "data_file",
|
||||
"fields": [
|
||||
{"name": "content", "type": "int"},
|
||||
{"name": "file_path", "type": "string"},
|
||||
{"name": "record_count", "type": "long"},
|
||||
{"name": "file_size_in_bytes", "type": "long"}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"#,
|
||||
)
|
||||
.expect("manifest avro schema should parse");
|
||||
let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest writer should initialize");
|
||||
for (file_path, content, status, snapshot_id, sequence_number) in files {
|
||||
writer
|
||||
.append_value(apache_avro::types::Value::Record(vec![
|
||||
("status".to_string(), apache_avro::types::Value::Int(*status)),
|
||||
("snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)),
|
||||
("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)),
|
||||
("file_sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)),
|
||||
(
|
||||
"data_file".to_string(),
|
||||
apache_avro::types::Value::Record(vec![
|
||||
("content".to_string(), apache_avro::types::Value::Int(*content)),
|
||||
("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())),
|
||||
("record_count".to_string(), apache_avro::types::Value::Long(1)),
|
||||
("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)),
|
||||
]),
|
||||
),
|
||||
]))
|
||||
.expect("manifest record should append");
|
||||
}
|
||||
writer.into_inner().expect("manifest avro bytes should flush")
|
||||
}
|
||||
|
||||
pub(crate) fn nullable_long(value: Option<i64>) -> apache_avro::types::Value {
|
||||
match value {
|
||||
Some(value) => apache_avro::types::Value::Union(1, Box::new(apache_avro::types::Value::Long(value))),
|
||||
None => apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn manifest_avro_bytes_with_nullable_sequences(files: &[(&str, i32, i32, i64, Option<i64>)]) -> Vec<u8> {
|
||||
let schema = apache_avro::Schema::parse_str(
|
||||
r#"
|
||||
{
|
||||
"type": "record",
|
||||
"name": "manifest_entry",
|
||||
"fields": [
|
||||
{"name": "status", "type": "int"},
|
||||
{"name": "snapshot_id", "type": "long"},
|
||||
{"name": "sequence_number", "type": ["null", "long"], "default": null},
|
||||
{"name": "file_sequence_number", "type": ["null", "long"], "default": null},
|
||||
{
|
||||
"name": "data_file",
|
||||
"type": {
|
||||
"type": "record",
|
||||
"name": "data_file",
|
||||
"fields": [
|
||||
{"name": "content", "type": "int"},
|
||||
{"name": "file_path", "type": "string"},
|
||||
{"name": "record_count", "type": "long"},
|
||||
{"name": "file_size_in_bytes", "type": "long"}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"#,
|
||||
)
|
||||
.expect("manifest avro schema should parse");
|
||||
let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest writer should initialize");
|
||||
for (file_path, content, status, snapshot_id, sequence_number) in files {
|
||||
writer
|
||||
.append_value(apache_avro::types::Value::Record(vec![
|
||||
("status".to_string(), apache_avro::types::Value::Int(*status)),
|
||||
("snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)),
|
||||
("sequence_number".to_string(), nullable_long(*sequence_number)),
|
||||
("file_sequence_number".to_string(), nullable_long(*sequence_number)),
|
||||
(
|
||||
"data_file".to_string(),
|
||||
apache_avro::types::Value::Record(vec![
|
||||
("content".to_string(), apache_avro::types::Value::Int(*content)),
|
||||
("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())),
|
||||
("record_count".to_string(), apache_avro::types::Value::Long(1)),
|
||||
("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)),
|
||||
]),
|
||||
),
|
||||
]))
|
||||
.expect("manifest record should append");
|
||||
}
|
||||
writer.into_inner().expect("manifest avro bytes should flush")
|
||||
}
|
||||
|
||||
// --- Stateful object backend shared by the store and admin handler tests
|
||||
// (backlog#1837 PR2). Superset instrumentation lands here incrementally;
|
||||
// this is the store-side fake moved verbatim.
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct TestCatalogObjectBackend {
|
||||
pub(crate) state: Arc<tokio::sync::Mutex<TestCatalogObjectState>>,
|
||||
pub(crate) locks: TestCatalogObjectLocks,
|
||||
pub(crate) strong_runtime: Option<StrongTableCatalogRuntime>,
|
||||
// One-shot, path-keyed injection knobs from the admin handler tests'
|
||||
// former TestTableCatalogObjectBackend (backlog#1837 PR2): each fires
|
||||
// once for the named object and clears itself, mirroring the original
|
||||
// semantics exactly. They compose with (and run before) the store tests'
|
||||
// attempt-indexed injection maps above.
|
||||
pub(crate) put_object_barrier: Option<Arc<tokio::sync::Barrier>>,
|
||||
pub(crate) fail_put_object_path: Arc<tokio::sync::Mutex<Option<String>>>,
|
||||
pub(crate) corrupt_put_object_path: Arc<tokio::sync::Mutex<Option<String>>>,
|
||||
pub(crate) missing_read_object_path: Arc<tokio::sync::Mutex<Option<String>>>,
|
||||
pub(crate) fail_read_object_path: Arc<tokio::sync::Mutex<Option<String>>>,
|
||||
pub(crate) lock_attempts: Arc<tokio::sync::Mutex<Vec<(String, String)>>>,
|
||||
/// Content-addressed (sha256) etags instead of the store fake's counter.
|
||||
/// The admin handler tests observe an object's etag and expect rewriting
|
||||
/// identical bytes to reproduce it, so their fixtures set this.
|
||||
pub(crate) content_addressed_etags: bool,
|
||||
}
|
||||
|
||||
pub(crate) type TestCatalogObjectLockKey = (String, String);
|
||||
pub(crate) type TestCatalogObjectLock = Arc<tokio::sync::RwLock<()>>;
|
||||
pub(crate) type TestCatalogObjectLocks = Arc<tokio::sync::Mutex<BTreeMap<TestCatalogObjectLockKey, TestCatalogObjectLock>>>;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct TestCatalogObjectPause {
|
||||
started: Arc<tokio::sync::Notify>,
|
||||
release: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl TestCatalogObjectPause {
|
||||
pub(crate) async fn wait_started(&self) {
|
||||
self.started.notified().await;
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BlockingObjectPublication {
|
||||
backend: TestCatalogObjectBackend,
|
||||
object: String,
|
||||
started: Arc<tokio::sync::Notify>,
|
||||
guard: Arc<parking_lot::Mutex<Option<Box<dyn Send>>>>,
|
||||
}
|
||||
|
||||
impl BlockingObjectPublication {
|
||||
pub(crate) fn new(backend: TestCatalogObjectBackend, object: impl Into<String>) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
object: object.into(),
|
||||
started: Arc::new(tokio::sync::Notify::new()),
|
||||
guard: Arc::new(parking_lot::Mutex::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_started(&self) {
|
||||
self.started.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct UnserializedTestPublication;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCommitPublication for UnserializedTestPublication {
|
||||
async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn prepare(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn holds_table_bucket(&self, _table_bucket: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn complete(&self) {}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCommitPublication for BlockingObjectPublication {
|
||||
async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn prepare(&self, table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> {
|
||||
self.started.notify_one();
|
||||
let guard = self.backend.acquire_read_lock(table_bucket, &self.object).await?;
|
||||
*self.guard.lock() = Some(guard);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn holds_table_bucket(&self, _table_bucket: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool {
|
||||
self.guard.lock().is_some()
|
||||
}
|
||||
|
||||
fn complete(&self) {
|
||||
drop(self.guard.lock().take());
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct TestCatalogObjectState {
|
||||
pub(crate) objects: BTreeMap<(String, String), TestCatalogObjectRecord>,
|
||||
pub(crate) etagless_objects: BTreeSet<(String, String)>,
|
||||
pub(crate) fail_read_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
pub(crate) pause_before_read_attempts: BTreeMap<(String, String), BTreeMap<usize, TestCatalogObjectPause>>,
|
||||
pub(crate) pause_read_attempts: BTreeMap<(String, String), BTreeMap<usize, TestCatalogObjectPause>>,
|
||||
pub(crate) read_attempts: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_limits: Vec<((String, String), usize)>,
|
||||
pub(crate) fail_put_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
pub(crate) fail_after_put_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
pub(crate) pause_put_attempts: BTreeMap<(String, String), BTreeMap<usize, TestCatalogObjectPause>>,
|
||||
pub(crate) fail_delete_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
pub(crate) fail_after_delete_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
pub(crate) put_attempts: BTreeMap<(String, String), usize>,
|
||||
pub(crate) delete_attempts: BTreeMap<(String, String), usize>,
|
||||
pub(crate) write_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_calls: usize,
|
||||
pub(crate) metadata_calls: usize,
|
||||
pub(crate) list_calls: usize,
|
||||
pub(crate) next_etag: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct TestCatalogObjectRecord {
|
||||
pub(crate) data: Vec<u8>,
|
||||
pub(crate) etag: String,
|
||||
pub(crate) mod_time: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
impl TestCatalogObjectBackend {
|
||||
pub(crate) async fn seed_object(&self, bucket: &str, object: &str, data: Vec<u8>) {
|
||||
self.seed_object_with_mod_time(bucket, object, data, Some(OffsetDateTime::UNIX_EPOCH))
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn seed_object_with_mod_time(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
) {
|
||||
let mut state = self.state.lock().await;
|
||||
let etag = state.next_etag();
|
||||
state
|
||||
.objects
|
||||
.insert((bucket.to_string(), object.to_string()), TestCatalogObjectRecord { data, etag, mod_time });
|
||||
}
|
||||
|
||||
pub(crate) async fn fail_put_attempt(&self, bucket: &str, object: &str, attempt: usize) {
|
||||
let mut state = self.state.lock().await;
|
||||
state
|
||||
.fail_put_attempts
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_default()
|
||||
.insert(attempt);
|
||||
}
|
||||
|
||||
pub(crate) async fn fail_delete_attempt(&self, bucket: &str, object: &str, attempt: usize) {
|
||||
let mut state = self.state.lock().await;
|
||||
state
|
||||
.fail_delete_attempts
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_default()
|
||||
.insert(attempt);
|
||||
}
|
||||
|
||||
pub(crate) async fn list_call_count(&self) -> usize {
|
||||
self.state.lock().await.list_calls
|
||||
}
|
||||
|
||||
pub(crate) async fn read_call_count(&self) -> usize {
|
||||
self.state.lock().await.read_calls
|
||||
}
|
||||
|
||||
pub(crate) async fn metadata_call_count(&self) -> usize {
|
||||
self.state.lock().await.metadata_calls
|
||||
}
|
||||
|
||||
pub(crate) async fn reset_call_counts(&self) {
|
||||
let mut state = self.state.lock().await;
|
||||
state.read_calls = 0;
|
||||
state.metadata_calls = 0;
|
||||
state.list_calls = 0;
|
||||
}
|
||||
|
||||
pub(crate) async fn write_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.write_lock_acquisitions
|
||||
.get(&(bucket.to_string(), object.to_string()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) async fn read_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.read_lock_acquisitions
|
||||
.get(&(bucket.to_string(), object.to_string()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) async fn fail_next_read(&self, bucket: &str, object: &str) {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
state.fail_read_attempts.entry(key).or_default().insert(next_attempt);
|
||||
}
|
||||
|
||||
pub(crate) async fn pause_next_read(&self, bucket: &str, object: &str) -> TestCatalogObjectPause {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
let pause = TestCatalogObjectPause::default();
|
||||
state
|
||||
.pause_read_attempts
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.insert(next_attempt, pause.clone());
|
||||
pause
|
||||
}
|
||||
|
||||
pub(crate) async fn pause_before_next_read(&self, bucket: &str, object: &str) -> TestCatalogObjectPause {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
let pause = TestCatalogObjectPause::default();
|
||||
state
|
||||
.pause_before_read_attempts
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.insert(next_attempt, pause.clone());
|
||||
pause
|
||||
}
|
||||
|
||||
pub(crate) async fn omit_etag_for_object(&self, bucket: &str, object: &str) {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.etagless_objects
|
||||
.insert((bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
pub(crate) async fn last_read_limit(&self, bucket: &str, object: &str) -> Option<usize> {
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.read_limits
|
||||
.iter()
|
||||
.rev()
|
||||
.find_map(|(read_key, limit)| (read_key == &key).then_some(*limit))
|
||||
}
|
||||
|
||||
pub(crate) async fn fail_next_put(&self, bucket: &str, object: &str) {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
state.fail_put_attempts.entry(key).or_default().insert(next_attempt);
|
||||
}
|
||||
|
||||
pub(crate) async fn fail_after_next_put(&self, bucket: &str, object: &str) {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
state.fail_after_put_attempts.entry(key).or_default().insert(next_attempt);
|
||||
}
|
||||
|
||||
pub(crate) async fn fail_after_next_delete(&self, bucket: &str, object: &str) {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.delete_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
state.fail_after_delete_attempts.entry(key).or_default().insert(next_attempt);
|
||||
}
|
||||
|
||||
pub(crate) async fn pause_next_put(&self, bucket: &str, object: &str) -> TestCatalogObjectPause {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
let pause = TestCatalogObjectPause::default();
|
||||
state
|
||||
.pause_put_attempts
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.insert(next_attempt, pause.clone());
|
||||
pause
|
||||
}
|
||||
|
||||
pub(crate) async fn put_attempt_count(&self, bucket: &str, object: &str) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.put_attempts
|
||||
.get(&(bucket.to_string(), object.to_string()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
impl TestCatalogObjectState {
|
||||
pub(crate) fn next_etag(&mut self) -> String {
|
||||
self.next_etag += 1;
|
||||
format!("etag-{}", self.next_etag)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
self.strong_runtime.clone()
|
||||
}
|
||||
|
||||
async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
|
||||
let mut missing_read_object_path = self.missing_read_object_path.lock().await;
|
||||
if missing_read_object_path.as_deref() == Some(object) {
|
||||
missing_read_object_path.take();
|
||||
return Ok(None);
|
||||
}
|
||||
drop(missing_read_object_path);
|
||||
|
||||
let mut fail_read_object_path = self.fail_read_object_path.lock().await;
|
||||
if fail_read_object_path.as_deref() == Some(object) {
|
||||
fail_read_object_path.take();
|
||||
return Err(TableCatalogStoreError::Internal("private generated metadata read failure".to_string()));
|
||||
}
|
||||
drop(fail_read_object_path);
|
||||
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let (attempt, pause_before) = {
|
||||
let mut state = self.state.lock().await;
|
||||
state.read_calls += 1;
|
||||
let attempt = {
|
||||
let attempts = state.read_attempts.entry(key.clone()).or_default();
|
||||
*attempts += 1;
|
||||
*attempts
|
||||
};
|
||||
if state
|
||||
.fail_read_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected read failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
let pause = state
|
||||
.pause_before_read_attempts
|
||||
.get_mut(&key)
|
||||
.and_then(|attempts| attempts.remove(&attempt));
|
||||
(attempt, pause)
|
||||
};
|
||||
if let Some(pause) = pause_before {
|
||||
pause.started.notify_one();
|
||||
pause.release.notified().await;
|
||||
}
|
||||
let (result, pause) = {
|
||||
let mut state = self.state.lock().await;
|
||||
let etagless = state.etagless_objects.contains(&key);
|
||||
let result = state.objects.get(&key).map(|record| TableCatalogObject {
|
||||
data: record.data.clone(),
|
||||
etag: (!etagless).then(|| record.etag.clone()),
|
||||
mod_time: record.mod_time,
|
||||
});
|
||||
let pause = state
|
||||
.pause_read_attempts
|
||||
.get_mut(&key)
|
||||
.and_then(|attempts| attempts.remove(&attempt));
|
||||
(result, pause)
|
||||
};
|
||||
if let Some(pause) = pause {
|
||||
pause.started.notify_one();
|
||||
pause.release.notified().await;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn read_object_limited(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
max_size: usize,
|
||||
) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.read_limits
|
||||
.push(((bucket.to_string(), object.to_string()), max_size));
|
||||
let result = self.read_object(bucket, object).await?;
|
||||
if result.as_ref().is_some_and(|object| object.data.len() > max_size) {
|
||||
return Err(TableCatalogStoreError::Invalid(format!(
|
||||
"catalog object {bucket}/{object} exceeds the maximum size of {max_size} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn object_metadata(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObjectMetadata>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.metadata_calls += 1;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let etagless = state.etagless_objects.contains(&key);
|
||||
Ok(state.objects.get(&key).map(|record| TableCatalogObjectMetadata {
|
||||
etag: (!etagless).then(|| record.etag.clone()),
|
||||
mod_time: record.mod_time,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<bool> {
|
||||
let state = self.state.lock().await;
|
||||
Ok(state.objects.contains_key(&(bucket.to_string(), object.to_string())))
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let mut fail_put_object_path = self.fail_put_object_path.lock().await;
|
||||
if fail_put_object_path.as_deref() == Some(object) {
|
||||
fail_put_object_path.take();
|
||||
return Err(TableCatalogStoreError::Internal("injected metadata write failure".to_string()));
|
||||
}
|
||||
drop(fail_put_object_path);
|
||||
|
||||
let mut corrupt_put_object_path = self.corrupt_put_object_path.lock().await;
|
||||
let data = if corrupt_put_object_path.as_deref() == Some(object) {
|
||||
corrupt_put_object_path.take();
|
||||
b"{}".to_vec()
|
||||
} else {
|
||||
data
|
||||
};
|
||||
drop(corrupt_put_object_path);
|
||||
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let (attempt, pause) = {
|
||||
let mut state = self.state.lock().await;
|
||||
let attempt = {
|
||||
let attempts = state.put_attempts.entry(key.clone()).or_default();
|
||||
*attempts += 1;
|
||||
*attempts
|
||||
};
|
||||
if state
|
||||
.fail_put_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected put failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
let pause = state
|
||||
.pause_put_attempts
|
||||
.get_mut(&key)
|
||||
.and_then(|attempts| attempts.remove(&attempt));
|
||||
(attempt, pause)
|
||||
};
|
||||
if let Some(pause) = pause {
|
||||
pause.started.notify_one();
|
||||
pause.release.notified().await;
|
||||
}
|
||||
|
||||
let result = {
|
||||
let mut state = self.state.lock().await;
|
||||
let precondition_failure = match &precondition {
|
||||
TableCatalogPutPrecondition::IfAbsent if state.objects.contains_key(&key) => {
|
||||
Some(TableCatalogStoreError::Conflict(format!("object already exists: {object}")))
|
||||
}
|
||||
TableCatalogPutPrecondition::IfMatch(expected) => match state.objects.get(&key) {
|
||||
None => Some(TableCatalogStoreError::Conflict(format!("object is missing: {object}"))),
|
||||
Some(current) if ¤t.etag != expected => {
|
||||
Some(TableCatalogStoreError::Conflict(format!("object changed: {object}")))
|
||||
}
|
||||
Some(_) => None,
|
||||
},
|
||||
_ => None,
|
||||
};
|
||||
if let Some(err) = precondition_failure {
|
||||
Err(err)
|
||||
} else {
|
||||
let etag = if self.content_addressed_etags {
|
||||
content_etag(&data)
|
||||
} else {
|
||||
state.next_etag()
|
||||
};
|
||||
state.objects.insert(
|
||||
key.clone(),
|
||||
TestCatalogObjectRecord {
|
||||
data,
|
||||
etag,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
},
|
||||
);
|
||||
if state
|
||||
.fail_after_put_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected post-commit put failure for {object} attempt {attempt}"
|
||||
)))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Some(barrier) = &self.put_object_barrier {
|
||||
barrier.wait().await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let attempt = {
|
||||
let attempts = state.delete_attempts.entry(key.clone()).or_default();
|
||||
*attempts += 1;
|
||||
*attempts
|
||||
};
|
||||
if state
|
||||
.fail_delete_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected delete failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
state.objects.remove(&key);
|
||||
if state
|
||||
.fail_after_delete_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected post-commit delete failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult<Vec<String>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.list_calls += 1;
|
||||
Ok(state
|
||||
.objects
|
||||
.keys()
|
||||
.filter(|(entry_bucket, object)| entry_bucket == bucket && object.starts_with(prefix))
|
||||
.map(|(_, object)| object.clone())
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Box<dyn Send>> {
|
||||
self.lock_attempts.lock().await.push((bucket.to_string(), object.to_string()));
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
*state
|
||||
.write_lock_acquisitions
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_default() += 1;
|
||||
}
|
||||
let lock = {
|
||||
let mut locks = self.locks.lock().await;
|
||||
locks
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(())))
|
||||
.clone()
|
||||
};
|
||||
Ok(Box::new(lock.write_owned().await))
|
||||
}
|
||||
|
||||
async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Box<dyn Send>> {
|
||||
// The admin fake implemented only acquire_write_lock, so the trait's
|
||||
// default read->write delegation made read acquisitions observable in
|
||||
// lock_attempts as well; keep that (backlog#1837 PR2).
|
||||
self.lock_attempts.lock().await.push((bucket.to_string(), object.to_string()));
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
*state
|
||||
.read_lock_acquisitions
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_default() += 1;
|
||||
}
|
||||
let lock = {
|
||||
let mut locks = self.locks.lock().await;
|
||||
locks
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(())))
|
||||
.clone()
|
||||
};
|
||||
Ok(Box::new(lock.read_owned().await))
|
||||
}
|
||||
}
|
||||
|
||||
fn content_etag(data: &[u8]) -> String {
|
||||
use sha2::Digest;
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(data), hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
/// Admin-handler-test conveniences carried over from the former
|
||||
/// TestTableCatalogObjectBackend (backlog#1837 PR2): content-addressed etags
|
||||
/// (sha256), direct record insertion, and lock observability.
|
||||
impl TestCatalogObjectBackend {
|
||||
/// Fake with the admin fixtures' content-addressed etag semantics.
|
||||
pub(crate) fn content_addressed() -> Self {
|
||||
Self {
|
||||
content_addressed_etags: true,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn put_bytes(&self, bucket: &str, object: &str, data: Vec<u8>) {
|
||||
let etag = content_etag(&data);
|
||||
self.state.lock().await.objects.insert(
|
||||
(bucket.to_string(), object.to_string()),
|
||||
TestCatalogObjectRecord {
|
||||
data,
|
||||
etag,
|
||||
mod_time: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) async fn put_json(&self, bucket: &str, object: &str, value: serde_json::Value) {
|
||||
self.put_json_with_mod_time(bucket, object, value, None).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn put_gzip_json(&self, bucket: &str, object: &str, value: serde_json::Value) {
|
||||
use std::io::Write;
|
||||
|
||||
let data = serde_json::to_vec(&value).expect("metadata JSON should serialize");
|
||||
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
encoder.write_all(&data).expect("metadata JSON should compress");
|
||||
self.put_bytes(bucket, object, encoder.finish().expect("metadata gzip stream should finish"))
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn put_json_with_mod_time(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
value: serde_json::Value,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
) {
|
||||
let data = serde_json::to_vec(&value).expect("metadata JSON should serialize");
|
||||
let etag = content_etag(&data);
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.objects
|
||||
.insert((bucket.to_string(), object.to_string()), TestCatalogObjectRecord { data, etag, mod_time });
|
||||
}
|
||||
|
||||
pub(crate) async fn write_lock_is_held(&self, bucket: &str, object: &str) -> bool {
|
||||
let lock = self
|
||||
.locks
|
||||
.lock()
|
||||
.await
|
||||
.get(&(bucket.to_string(), object.to_string()))
|
||||
.cloned();
|
||||
lock.is_some_and(|lock| lock.try_write_owned().is_err())
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_lock_attempts(&self, count: usize) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
loop {
|
||||
if self.lock_attempts.lock().await.len() >= count {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("lock acquisition attempts should be observable");
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ use super::identifier::{
|
||||
default_table_lifecycle_path, default_table_marker_path, default_table_root_prefix, is_valid_table_metadata_file_name,
|
||||
namespace_name_from_marker_path, table_name_from_marker_path, validate_object_mutation,
|
||||
};
|
||||
use super::test_support::{BlockingObjectPublication, TestCatalogObjectBackend, UnserializedTestPublication};
|
||||
use super::*;
|
||||
use datafusion::{
|
||||
arrow::{
|
||||
@@ -1077,312 +1078,6 @@ fn catalog_object_entry_paths_use_internal_root_and_hashed_untrusted_ids() {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct TestCatalogObjectBackend {
|
||||
state: Arc<tokio::sync::Mutex<TestCatalogObjectState>>,
|
||||
locks: TestCatalogObjectLocks,
|
||||
strong_runtime: Option<StrongTableCatalogRuntime>,
|
||||
}
|
||||
|
||||
type TestCatalogObjectLockKey = (String, String);
|
||||
type TestCatalogObjectLock = Arc<tokio::sync::RwLock<()>>;
|
||||
type TestCatalogObjectLocks = Arc<tokio::sync::Mutex<BTreeMap<TestCatalogObjectLockKey, TestCatalogObjectLock>>>;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct TestCatalogObjectPause {
|
||||
started: Arc<tokio::sync::Notify>,
|
||||
release: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl TestCatalogObjectPause {
|
||||
async fn wait_started(&self) {
|
||||
self.started.notified().await;
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
self.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BlockingObjectPublication {
|
||||
backend: TestCatalogObjectBackend,
|
||||
object: String,
|
||||
started: Arc<tokio::sync::Notify>,
|
||||
guard: Arc<parking_lot::Mutex<Option<Box<dyn Send>>>>,
|
||||
}
|
||||
|
||||
impl BlockingObjectPublication {
|
||||
fn new(backend: TestCatalogObjectBackend, object: impl Into<String>) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
object: object.into(),
|
||||
started: Arc::new(tokio::sync::Notify::new()),
|
||||
guard: Arc::new(parking_lot::Mutex::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_started(&self) {
|
||||
self.started.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct UnserializedTestPublication;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCommitPublication for UnserializedTestPublication {
|
||||
async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn prepare(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn holds_table_bucket(&self, _table_bucket: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn complete(&self) {}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCommitPublication for BlockingObjectPublication {
|
||||
async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn prepare(&self, table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> {
|
||||
self.started.notify_one();
|
||||
let guard = self.backend.acquire_read_lock(table_bucket, &self.object).await?;
|
||||
*self.guard.lock() = Some(guard);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn holds_table_bucket(&self, _table_bucket: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool {
|
||||
self.guard.lock().is_some()
|
||||
}
|
||||
|
||||
fn complete(&self) {
|
||||
drop(self.guard.lock().take());
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TestCatalogObjectState {
|
||||
objects: BTreeMap<(String, String), TestCatalogObjectRecord>,
|
||||
etagless_objects: BTreeSet<(String, String)>,
|
||||
fail_read_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
pause_before_read_attempts: BTreeMap<(String, String), BTreeMap<usize, TestCatalogObjectPause>>,
|
||||
pause_read_attempts: BTreeMap<(String, String), BTreeMap<usize, TestCatalogObjectPause>>,
|
||||
read_attempts: BTreeMap<(String, String), usize>,
|
||||
read_limits: Vec<((String, String), usize)>,
|
||||
fail_put_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
fail_after_put_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
pause_put_attempts: BTreeMap<(String, String), BTreeMap<usize, TestCatalogObjectPause>>,
|
||||
fail_delete_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
fail_after_delete_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
|
||||
put_attempts: BTreeMap<(String, String), usize>,
|
||||
delete_attempts: BTreeMap<(String, String), usize>,
|
||||
write_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
read_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
read_calls: usize,
|
||||
metadata_calls: usize,
|
||||
list_calls: usize,
|
||||
next_etag: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestCatalogObjectRecord {
|
||||
data: Vec<u8>,
|
||||
etag: String,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
impl TestCatalogObjectBackend {
|
||||
async fn seed_object(&self, bucket: &str, object: &str, data: Vec<u8>) {
|
||||
self.seed_object_with_mod_time(bucket, object, data, Some(OffsetDateTime::UNIX_EPOCH))
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn seed_object_with_mod_time(&self, bucket: &str, object: &str, data: Vec<u8>, mod_time: Option<OffsetDateTime>) {
|
||||
let mut state = self.state.lock().await;
|
||||
let etag = state.next_etag();
|
||||
state
|
||||
.objects
|
||||
.insert((bucket.to_string(), object.to_string()), TestCatalogObjectRecord { data, etag, mod_time });
|
||||
}
|
||||
|
||||
async fn fail_put_attempt(&self, bucket: &str, object: &str, attempt: usize) {
|
||||
let mut state = self.state.lock().await;
|
||||
state
|
||||
.fail_put_attempts
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_default()
|
||||
.insert(attempt);
|
||||
}
|
||||
|
||||
async fn fail_delete_attempt(&self, bucket: &str, object: &str, attempt: usize) {
|
||||
let mut state = self.state.lock().await;
|
||||
state
|
||||
.fail_delete_attempts
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_default()
|
||||
.insert(attempt);
|
||||
}
|
||||
|
||||
async fn list_call_count(&self) -> usize {
|
||||
self.state.lock().await.list_calls
|
||||
}
|
||||
|
||||
async fn read_call_count(&self) -> usize {
|
||||
self.state.lock().await.read_calls
|
||||
}
|
||||
|
||||
async fn metadata_call_count(&self) -> usize {
|
||||
self.state.lock().await.metadata_calls
|
||||
}
|
||||
|
||||
async fn reset_call_counts(&self) {
|
||||
let mut state = self.state.lock().await;
|
||||
state.read_calls = 0;
|
||||
state.metadata_calls = 0;
|
||||
state.list_calls = 0;
|
||||
}
|
||||
|
||||
async fn write_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.write_lock_acquisitions
|
||||
.get(&(bucket.to_string(), object.to_string()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
async fn read_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.read_lock_acquisitions
|
||||
.get(&(bucket.to_string(), object.to_string()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
async fn fail_next_read(&self, bucket: &str, object: &str) {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
state.fail_read_attempts.entry(key).or_default().insert(next_attempt);
|
||||
}
|
||||
|
||||
async fn pause_next_read(&self, bucket: &str, object: &str) -> TestCatalogObjectPause {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
let pause = TestCatalogObjectPause::default();
|
||||
state
|
||||
.pause_read_attempts
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.insert(next_attempt, pause.clone());
|
||||
pause
|
||||
}
|
||||
|
||||
async fn pause_before_next_read(&self, bucket: &str, object: &str) -> TestCatalogObjectPause {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
let pause = TestCatalogObjectPause::default();
|
||||
state
|
||||
.pause_before_read_attempts
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.insert(next_attempt, pause.clone());
|
||||
pause
|
||||
}
|
||||
|
||||
async fn omit_etag_for_object(&self, bucket: &str, object: &str) {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.etagless_objects
|
||||
.insert((bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
async fn last_read_limit(&self, bucket: &str, object: &str) -> Option<usize> {
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.read_limits
|
||||
.iter()
|
||||
.rev()
|
||||
.find_map(|(read_key, limit)| (read_key == &key).then_some(*limit))
|
||||
}
|
||||
|
||||
async fn fail_next_put(&self, bucket: &str, object: &str) {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
state.fail_put_attempts.entry(key).or_default().insert(next_attempt);
|
||||
}
|
||||
|
||||
async fn fail_after_next_put(&self, bucket: &str, object: &str) {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
state.fail_after_put_attempts.entry(key).or_default().insert(next_attempt);
|
||||
}
|
||||
|
||||
async fn fail_after_next_delete(&self, bucket: &str, object: &str) {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.delete_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
state.fail_after_delete_attempts.entry(key).or_default().insert(next_attempt);
|
||||
}
|
||||
|
||||
async fn pause_next_put(&self, bucket: &str, object: &str) -> TestCatalogObjectPause {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
let pause = TestCatalogObjectPause::default();
|
||||
state
|
||||
.pause_put_attempts
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.insert(next_attempt, pause.clone());
|
||||
pause
|
||||
}
|
||||
|
||||
async fn put_attempt_count(&self, bucket: &str, object: &str) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.put_attempts
|
||||
.get(&(bucket.to_string(), object.to_string()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
impl TestCatalogObjectState {
|
||||
fn next_etag(&mut self) -> String {
|
||||
self.next_etag += 1;
|
||||
format!("etag-{}", self.next_etag)
|
||||
}
|
||||
}
|
||||
|
||||
fn maintenance_object_report<'a>(
|
||||
report: &'a TableMetadataMaintenanceReport,
|
||||
metadata_location: &str,
|
||||
@@ -1429,54 +1124,12 @@ fn manifest_list_avro_bytes(manifest_paths: &[&str]) -> Vec<u8> {
|
||||
}
|
||||
|
||||
fn manifest_list_avro_bytes_with_spec(manifest_paths: &[&str], partition_spec_id: i32) -> Vec<u8> {
|
||||
let schema = apache_avro::Schema::parse_str(
|
||||
r#"
|
||||
{
|
||||
"type": "record",
|
||||
"name": "manifest_file",
|
||||
"fields": [
|
||||
{"name": "manifest_path", "type": "string"},
|
||||
{"name": "manifest_length", "type": "long"},
|
||||
{"name": "partition_spec_id", "type": "int"},
|
||||
{"name": "content", "type": "int"},
|
||||
{"name": "sequence_number", "type": "long"},
|
||||
{"name": "min_sequence_number", "type": "long"},
|
||||
{"name": "added_snapshot_id", "type": "long"},
|
||||
{"name": "added_files_count", "type": "int"},
|
||||
{"name": "existing_files_count", "type": "int"},
|
||||
{"name": "deleted_files_count", "type": "int"},
|
||||
{"name": "added_rows_count", "type": "long"},
|
||||
{"name": "existing_rows_count", "type": "long"},
|
||||
{"name": "deleted_rows_count", "type": "long"}
|
||||
]
|
||||
}
|
||||
"#,
|
||||
)
|
||||
.expect("manifest list avro schema should parse");
|
||||
let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest list writer should initialize");
|
||||
for manifest_path in manifest_paths {
|
||||
writer
|
||||
.append_value(apache_avro::types::Value::Record(vec![
|
||||
(
|
||||
"manifest_path".to_string(),
|
||||
apache_avro::types::Value::String((*manifest_path).to_string()),
|
||||
),
|
||||
("manifest_length".to_string(), apache_avro::types::Value::Long(1)),
|
||||
("partition_spec_id".to_string(), apache_avro::types::Value::Int(partition_spec_id)),
|
||||
("content".to_string(), apache_avro::types::Value::Int(0)),
|
||||
("sequence_number".to_string(), apache_avro::types::Value::Long(7)),
|
||||
("min_sequence_number".to_string(), apache_avro::types::Value::Long(7)),
|
||||
("added_snapshot_id".to_string(), apache_avro::types::Value::Long(20)),
|
||||
("added_files_count".to_string(), apache_avro::types::Value::Int(1)),
|
||||
("existing_files_count".to_string(), apache_avro::types::Value::Int(0)),
|
||||
("deleted_files_count".to_string(), apache_avro::types::Value::Int(0)),
|
||||
("added_rows_count".to_string(), apache_avro::types::Value::Long(1)),
|
||||
("existing_rows_count".to_string(), apache_avro::types::Value::Long(0)),
|
||||
("deleted_rows_count".to_string(), apache_avro::types::Value::Long(0)),
|
||||
]))
|
||||
.expect("manifest list record should append");
|
||||
}
|
||||
writer.into_inner().expect("manifest list avro bytes should flush")
|
||||
// Historical fixed values of this file's fixtures: sequence 7, snapshot 20.
|
||||
let manifests = manifest_paths
|
||||
.iter()
|
||||
.map(|path| (*path, partition_spec_id, 7_i64, 20_i64))
|
||||
.collect::<Vec<_>>();
|
||||
crate::table_catalog::test_support::manifest_list_avro_entries_with_partition_specs(&manifests)
|
||||
}
|
||||
|
||||
fn v1_manifest_list_avro_bytes(manifest_path: &str) -> Vec<u8> {
|
||||
@@ -2336,55 +1989,13 @@ fn manifest_avro_bytes(files: &[(&str, i32)]) -> Vec<u8> {
|
||||
}
|
||||
|
||||
fn manifest_avro_bytes_with_status(files: &[(&str, i32, i32)]) -> Vec<u8> {
|
||||
let schema = apache_avro::Schema::parse_str(
|
||||
r#"
|
||||
{
|
||||
"type": "record",
|
||||
"name": "manifest_entry",
|
||||
"fields": [
|
||||
{"name": "status", "type": "int"},
|
||||
{"name": "snapshot_id", "type": "long"},
|
||||
{"name": "sequence_number", "type": "long"},
|
||||
{"name": "file_sequence_number", "type": "long"},
|
||||
{
|
||||
"name": "data_file",
|
||||
"type": {
|
||||
"type": "record",
|
||||
"name": "data_file",
|
||||
"fields": [
|
||||
{"name": "content", "type": "int"},
|
||||
{"name": "file_path", "type": "string"},
|
||||
{"name": "record_count", "type": "long"},
|
||||
{"name": "file_size_in_bytes", "type": "long"}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"#,
|
||||
)
|
||||
.expect("manifest avro schema should parse");
|
||||
let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest writer should initialize");
|
||||
for (file_path, content, status) in files {
|
||||
writer
|
||||
.append_value(apache_avro::types::Value::Record(vec![
|
||||
("status".to_string(), apache_avro::types::Value::Int(*status)),
|
||||
("snapshot_id".to_string(), apache_avro::types::Value::Long(20)),
|
||||
("sequence_number".to_string(), apache_avro::types::Value::Long(7)),
|
||||
("file_sequence_number".to_string(), apache_avro::types::Value::Long(7)),
|
||||
(
|
||||
"data_file".to_string(),
|
||||
apache_avro::types::Value::Record(vec![
|
||||
("content".to_string(), apache_avro::types::Value::Int(*content)),
|
||||
("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())),
|
||||
("record_count".to_string(), apache_avro::types::Value::Long(1)),
|
||||
("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)),
|
||||
]),
|
||||
),
|
||||
]))
|
||||
.expect("manifest record should append");
|
||||
}
|
||||
writer.into_inner().expect("manifest avro bytes should flush")
|
||||
// Historical fixed values of this file's fixtures: snapshot 20, sequence 7
|
||||
// (the shared constructor takes snapshot_id fourth, sequence fifth).
|
||||
let files = files
|
||||
.iter()
|
||||
.map(|(path, content, status)| (*path, *content, *status, 20_i64, 7_i64))
|
||||
.collect::<Vec<_>>();
|
||||
crate::table_catalog::test_support::manifest_avro_bytes(&files)
|
||||
}
|
||||
|
||||
fn manifest_avro_bytes_with_dt_partition(files: &[(&str, i32, &str)]) -> Vec<u8> {
|
||||
@@ -2550,248 +2161,6 @@ fn parquet_i32_values(data: Vec<u8>) -> Vec<i32> {
|
||||
values
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
self.strong_runtime.clone()
|
||||
}
|
||||
|
||||
async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let (attempt, pause_before) = {
|
||||
let mut state = self.state.lock().await;
|
||||
state.read_calls += 1;
|
||||
let attempt = {
|
||||
let attempts = state.read_attempts.entry(key.clone()).or_default();
|
||||
*attempts += 1;
|
||||
*attempts
|
||||
};
|
||||
if state
|
||||
.fail_read_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected read failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
let pause = state
|
||||
.pause_before_read_attempts
|
||||
.get_mut(&key)
|
||||
.and_then(|attempts| attempts.remove(&attempt));
|
||||
(attempt, pause)
|
||||
};
|
||||
if let Some(pause) = pause_before {
|
||||
pause.started.notify_one();
|
||||
pause.release.notified().await;
|
||||
}
|
||||
let (result, pause) = {
|
||||
let mut state = self.state.lock().await;
|
||||
let etagless = state.etagless_objects.contains(&key);
|
||||
let result = state.objects.get(&key).map(|record| TableCatalogObject {
|
||||
data: record.data.clone(),
|
||||
etag: (!etagless).then(|| record.etag.clone()),
|
||||
mod_time: record.mod_time,
|
||||
});
|
||||
let pause = state
|
||||
.pause_read_attempts
|
||||
.get_mut(&key)
|
||||
.and_then(|attempts| attempts.remove(&attempt));
|
||||
(result, pause)
|
||||
};
|
||||
if let Some(pause) = pause {
|
||||
pause.started.notify_one();
|
||||
pause.release.notified().await;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn read_object_limited(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
max_size: usize,
|
||||
) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.read_limits
|
||||
.push(((bucket.to_string(), object.to_string()), max_size));
|
||||
let result = self.read_object(bucket, object).await?;
|
||||
if result.as_ref().is_some_and(|object| object.data.len() > max_size) {
|
||||
return Err(TableCatalogStoreError::Invalid(format!(
|
||||
"catalog object {bucket}/{object} exceeds the maximum size of {max_size} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn object_metadata(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObjectMetadata>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.metadata_calls += 1;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let etagless = state.etagless_objects.contains(&key);
|
||||
Ok(state.objects.get(&key).map(|record| TableCatalogObjectMetadata {
|
||||
etag: (!etagless).then(|| record.etag.clone()),
|
||||
mod_time: record.mod_time,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<bool> {
|
||||
let state = self.state.lock().await;
|
||||
Ok(state.objects.contains_key(&(bucket.to_string(), object.to_string())))
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let (attempt, pause) = {
|
||||
let mut state = self.state.lock().await;
|
||||
let attempt = {
|
||||
let attempts = state.put_attempts.entry(key.clone()).or_default();
|
||||
*attempts += 1;
|
||||
*attempts
|
||||
};
|
||||
if state
|
||||
.fail_put_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected put failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
let pause = state
|
||||
.pause_put_attempts
|
||||
.get_mut(&key)
|
||||
.and_then(|attempts| attempts.remove(&attempt));
|
||||
(attempt, pause)
|
||||
};
|
||||
if let Some(pause) = pause {
|
||||
pause.started.notify_one();
|
||||
pause.release.notified().await;
|
||||
}
|
||||
|
||||
let mut state = self.state.lock().await;
|
||||
match precondition {
|
||||
TableCatalogPutPrecondition::IfAbsent if state.objects.contains_key(&key) => {
|
||||
return Err(TableCatalogStoreError::Conflict(format!("object already exists: {object}")));
|
||||
}
|
||||
TableCatalogPutPrecondition::IfMatch(expected) => {
|
||||
let Some(current) = state.objects.get(&key) else {
|
||||
return Err(TableCatalogStoreError::Conflict(format!("object is missing: {object}")));
|
||||
};
|
||||
if current.etag != expected {
|
||||
return Err(TableCatalogStoreError::Conflict(format!("object changed: {object}")));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let etag = state.next_etag();
|
||||
state.objects.insert(
|
||||
key.clone(),
|
||||
TestCatalogObjectRecord {
|
||||
data,
|
||||
etag,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
},
|
||||
);
|
||||
if state
|
||||
.fail_after_put_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected post-commit put failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let attempt = {
|
||||
let attempts = state.delete_attempts.entry(key.clone()).or_default();
|
||||
*attempts += 1;
|
||||
*attempts
|
||||
};
|
||||
if state
|
||||
.fail_delete_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected delete failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
state.objects.remove(&key);
|
||||
if state
|
||||
.fail_after_delete_attempts
|
||||
.get(&key)
|
||||
.is_some_and(|attempts| attempts.contains(&attempt))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Internal(format!(
|
||||
"injected post-commit delete failure for {object} attempt {attempt}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult<Vec<String>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.list_calls += 1;
|
||||
Ok(state
|
||||
.objects
|
||||
.keys()
|
||||
.filter(|(entry_bucket, object)| entry_bucket == bucket && object.starts_with(prefix))
|
||||
.map(|(_, object)| object.clone())
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Box<dyn Send>> {
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
*state
|
||||
.write_lock_acquisitions
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_default() += 1;
|
||||
}
|
||||
let lock = {
|
||||
let mut locks = self.locks.lock().await;
|
||||
locks
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(())))
|
||||
.clone()
|
||||
};
|
||||
Ok(Box::new(lock.write_owned().await))
|
||||
}
|
||||
|
||||
async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Box<dyn Send>> {
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
*state
|
||||
.read_lock_acquisitions
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_default() += 1;
|
||||
}
|
||||
let lock = {
|
||||
let mut locks = self.locks.lock().await;
|
||||
locks
|
||||
.entry((bucket.to_string(), object.to_string()))
|
||||
.or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(())))
|
||||
.clone()
|
||||
};
|
||||
Ok(Box::new(lock.read_owned().await))
|
||||
}
|
||||
}
|
||||
|
||||
fn test_bucket_entry(bucket: &str) -> TableBucketEntry {
|
||||
TableBucketEntry {
|
||||
version: TABLE_CATALOG_ENTRY_VERSION,
|
||||
|
||||
Reference in New Issue
Block a user