Compare commits

...

25 Commits

Author SHA1 Message Date
overtrue cfe75fddd9 fix(ecstore): preserve remote preparation corruption 2026-10-16 05:56:35 +08:00
overtrue fe7854500c test(ecstore): reject conflicting native preparation locators 2026-10-15 16:30:27 +08:00
overtrue 75ab3052a9 fix(ecstore): publish native stream files atomically 2026-10-15 16:14:01 +08:00
overtrue 368cdfd214 test(storage): cover remote prep restart 2026-10-15 13:38:03 +08:00
overtrue 90a1eb814e feat(ecstore): add authenticated native prep disk RPCs 2026-10-15 12:56:25 +08:00
overtrue b97a952cda test(ecstore): verify saved native disk set order 2026-10-15 12:18:04 +08:00
overtrue 0a8e78238b fix(ecstore): restore native preparation disk dispatch 2026-10-15 12:09:10 +08:00
overtrue fe04c79894 feat(capture-authority): mint exact committed Created authorization 2026-10-14 05:54:24 +08:00
overtrue 5c4a285a7b fix(ecstore): reject split native preparation stream paths 2026-10-14 05:54:12 +08:00
overtrue 9a8faf7c86 feat(ecstore): persist streaming native object preparations 2026-10-14 05:45:55 +08:00
overtrue 0a2f58c7a9 feat(capture): wire native Raft authority into startup 2026-10-14 04:43:26 +08:00
overtrue 4d97501327 feat(ecstore): add authenticated native Raft TCP transport 2026-10-14 03:42:24 +08:00
overtrue 3b2af12c64 feat(capture-authority): add real native Raft owner lifecycle 2026-10-14 02:47:52 +08:00
overtrue c73ddde848 feat(capture-authority): add durable Raft state machine snapshots 2026-10-14 02:30:43 +08:00
overtrue ef72a37fd7 feat(capture): adapt Raft log storage and authorize prefix purge 2026-10-14 02:06:07 +08:00
overtrue e46cf750dd feat(capture): expose bounded local native source reader 2026-10-14 01:21:11 +08:00
overtrue fe955fa3ee feat(iam): issue plugin credentials from original-user delegations 2026-10-13 22:16:46 +08:00
overtrue 64578c067e feat(capture): persist local checkpoints and compact covered logs 2026-10-13 05:03:29 +08:00
overtrue 78715f0921 feat(ecstore): add durable native object preparation
Encode actual reader data into private local shards and recover matching write-quorum receipts. Preserve original metadata and namespace owners across retries and cancellation, without granting publication authority.

Validate with 13 focused tests and two rejected digest/quorum mutations. Document the preparation and recovery boundary.
2026-10-13 03:56:13 +08:00
overtrue 3284aafd73 feat(ecstore): persist private native preparation receipts 2026-10-13 01:02:05 +08:00
overtrue 9286442f87 feat(capture): add rebuildable committed event pagination 2026-10-12 08:31:29 +08:00
overtrue b8aad1c168 fix(capture): validate checkpoint cut against captured committed log 2026-10-11 23:48:08 +08:00
overtrue 5fcb229760 feat(ecstore): observe bounded config under retained write owner 2026-10-11 22:59:16 +08:00
overtrue 94baef7462 feat(capture): add explicit schema-bound checkpoint capabilities 2026-10-11 21:30:55 +08:00
overtrue 60d90f085a feat(ecstore): add bounded persisted config observations 2026-10-11 20:29:27 +08:00
89 changed files with 22597 additions and 164 deletions
Generated
+5
View File
@@ -9672,6 +9672,7 @@ dependencies = [
"mime_guess",
"moka",
"object 0.40.0",
"openraft",
"opentelemetry",
"opentelemetry-proto",
"opentelemetry_sdk",
@@ -9692,6 +9693,7 @@ dependencies = [
"rsa 0.10.0-rc.18",
"rust-embed",
"rustfs-audit",
"rustfs-capture-authority",
"rustfs-common",
"rustfs-concurrency",
"rustfs-config",
@@ -9806,6 +9808,7 @@ version = "1.0.1"
dependencies = [
"openraft",
"redb",
"rustix",
"serde",
"serde_json",
"sha2 0.11.0",
@@ -9970,6 +9973,7 @@ dependencies = [
"metrics-util",
"moka",
"num_cpus",
"openraft",
"opentelemetry",
"opentelemetry_sdk",
"parking_lot",
@@ -9984,6 +9988,7 @@ dependencies = [
"reqwest",
"rmp",
"rmp-serde",
"rustfs-capture-authority",
"rustfs-common",
"rustfs-concurrency",
"rustfs-config",
+1
View File
@@ -104,6 +104,7 @@ rustfs-config = { path = "./crates/config", version = "1.0.1" }
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
rustfs-capture-authority = { path = "crates/capture-authority", version = "1.0.1" }
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
rustfs-iam = { path = "crates/iam", version = "1.0.1" }
+8 -4
View File
@@ -8,17 +8,21 @@ repository.workspace = true
description = "Durable local storage for capture authority"
[dependencies]
openraft = { workspace = true }
openraft = { workspace = true, features = ["storage-v2"] }
redb = { workspace = true }
sha2 = { workspace = true }
serde = { workspace = true, features = ["derive"] }
thiserror = { workspace = true }
tokio = { workspace = true }
tokio = { workspace = true, features = ["net", "io-util", "time", "sync", "rt"] }
url = { workspace = true }
tempfile = { workspace = true }
serde_json = { workspace = true }
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
rustix = { workspace = true, features = ["process"] }
[dev-dependencies]
serde_json = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
[lints]
workspace = true
+174 -30
View File
@@ -48,12 +48,12 @@ pub(crate) const EVENTS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("e
pub(crate) const OPERATIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("operations");
pub(crate) const APPLY_RESULTS: TableDefinition<u64, &[u8]> = TableDefinition::new("apply_results");
fn version_key(object: &ObjectIdentity, version: [u8; 16]) -> Result<Vec<u8>, StoreError> {
pub(crate) fn version_key(object: &ObjectIdentity, version: [u8; 16]) -> Result<Vec<u8>, StoreError> {
let mut b = object_bytes(object)?;
b.extend(version);
Ok(b)
}
fn decode_version(b: &[u8]) -> Result<CreatedVersion, StoreError> {
pub(crate) fn decode_version(b: &[u8]) -> Result<CreatedVersion, StoreError> {
let mut d = Decoder::new(b);
let v = CreatedVersion {
prepared: d.prepared()?,
@@ -63,7 +63,7 @@ fn decode_version(b: &[u8]) -> Result<CreatedVersion, StoreError> {
d.finish()?;
Ok(v)
}
fn decode_event(b: &[u8]) -> Result<CreatedEvent, StoreError> {
pub(crate) fn decode_event(b: &[u8]) -> Result<CreatedEvent, StoreError> {
let mut d = Decoder::new(b);
let e = CreatedEvent {
object: d.object()?,
@@ -89,7 +89,7 @@ fn decode_operation(b: &[u8]) -> Result<CreatedOperation, StoreError> {
result,
})
}
fn decode_provenance(b: &[u8]) -> Result<CreatedOperationProvenance, StoreError> {
pub(crate) fn decode_provenance(b: &[u8]) -> Result<CreatedOperationProvenance, StoreError> {
if b.len() > MAX_OBJECT_KEY_LENGTH + 512 || b.get(..5) != Some(b"COPR\x01") {
return Err(StoreError::CorruptRecord);
}
@@ -103,15 +103,19 @@ fn decode_provenance(b: &[u8]) -> Result<CreatedOperationProvenance, StoreError>
})
}
fn decode_stored_operation(b: &[u8], schema: u64) -> Result<CreatedOperation, StoreError> {
if matches!(schema, 5 | 6) {
pub(crate) fn validate_provenance_encoding(bytes: &[u8]) -> Result<(), StoreError> {
decode_provenance(bytes).map(|_| ())
}
pub(crate) fn decode_stored_operation(b: &[u8], schema: u64) -> Result<CreatedOperation, StoreError> {
if matches!(schema, 5 | 6 | 7) {
Ok(decode_provenance(b)?.operation)
} else {
decode_operation(b)
}
}
fn encode_provenance(first: LogId, operation: &[u8]) -> Result<Vec<u8>, StoreError> {
pub(crate) fn encode_provenance(first: LogId, operation: &[u8]) -> Result<Vec<u8>, StoreError> {
let mut bytes = b"COPR\x01".to_vec();
bytes.extend(encode_id(first));
bytes.extend(operation);
@@ -135,7 +139,7 @@ pub(crate) fn read_binding(meta: &impl ReadableTable<&'static str, &'static [u8]
d.finish()?;
Ok(binding)
}
fn read_applied(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
pub(crate) fn read_applied(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
let b = meta.get("last_applied")?.ok_or(StoreError::CorruptRecord)?;
let mut d = Decoder::new(b.value());
let id = match d.take::<1>()?[0] {
@@ -200,7 +204,7 @@ pub(crate) fn decode_native_entry(entry: crate::LogEntry) -> Result<crate::Captu
})
}
fn apply_created(
pub(crate) fn apply_created(
tx: &redb::WriteTransaction,
id: LogId,
command: &DecideCreated,
@@ -251,6 +255,7 @@ fn apply_created(
event.extend(binding_bytes(&binding)?);
tx.open_table(EVENTS)?
.insert(command.operation_id.as_slice(), event.as_slice())?;
crate::source_page::insert_event_index(tx, id, command.operation_id)?;
heads.insert(object_bytes(&command.object)?.as_slice(), command.prepared.version_id.as_slice())?;
CreatedResult::Created {
version_id: command.prepared.version_id,
@@ -264,7 +269,7 @@ fn apply_created(
.to_vec();
op.extend(canonical);
op.extend(result_bytes(&outcome));
if matches!(schema, 5 | 6) {
if matches!(schema, 5 | 6 | 7) {
op = encode_provenance(id, &op)?;
}
operations.insert(command.operation_id.as_slice(), op.as_slice())?;
@@ -273,7 +278,10 @@ fn apply_created(
Ok(result)
}
fn validate_response_kind(entry: &crate::CaptureRaftEntry, response: &crate::CaptureResponse) -> Result<(), StoreError> {
pub(crate) fn validate_response_kind(
entry: &crate::CaptureRaftEntry,
response: &crate::CaptureResponse,
) -> Result<(), StoreError> {
let kind = match entry.payload {
openraft::EntryPayload::Blank => crate::CaptureResponseKind::Blank,
openraft::EntryPayload::Normal(_) => crate::CaptureResponseKind::Created,
@@ -301,7 +309,7 @@ fn encode_response(response: &crate::CaptureResponse) -> Result<Vec<u8>, StoreEr
bytes.extend(body);
Ok(bytes)
}
fn decode_response(bytes: &[u8]) -> Result<crate::CaptureResponse, StoreError> {
pub(crate) fn decode_response(bytes: &[u8]) -> Result<crate::CaptureResponse, StoreError> {
if bytes.len() < 38
|| &bytes[..5] != b"CRSP\x01"
|| number(&bytes[30..38])? != u64::try_from(bytes.len() - 38).map_err(|_| StoreError::CorruptRecord)?
@@ -338,12 +346,16 @@ pub(crate) fn native_application_state(tx: &ReadTransaction) -> Result<(Option<L
fn native_application_metadata(
meta: &impl ReadableTable<&'static str, &'static [u8]>,
) -> Result<(Option<LogId>, crate::CaptureMembership), StoreError> {
if !matches!(number(required(meta.get("schema")?)?.value())?, 4 | 6) {
let schema = number(required(meta.get("schema")?)?.value())?;
if !matches!(schema, 4 | 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
let last = read_applied(meta)?;
let membership = crate::CaptureMembership::decode(required(meta.get("native_membership")?)?.value())
.map_err(|_| StoreError::CorruptRecord)?;
let membership_bytes = required(meta.get("native_membership")?)?;
if schema == 7 && membership_bytes.value().len() > 1024 * 1024 {
return Err(StoreError::CorruptRecord);
}
let membership = crate::CaptureMembership::decode(membership_bytes.value()).map_err(|_| StoreError::CorruptRecord)?;
if membership
.log_id()
.is_some_and(|id| last.is_none_or(|last| id.index > last.index))
@@ -417,11 +429,11 @@ impl Store {
{
let mut meta = tx.open_table(META)?;
let schema = number(required(meta.get("schema")?)?.value())?;
if !matches!(schema, 3..=6) {
if !matches!(schema, 3..=7) {
return Err(StoreError::UnsupportedSchema);
}
read_binding(&meta)?;
if matches!(schema, 4 | 6) {
if matches!(schema, 4 | 6 | 7) {
let membership = crate::CaptureMembership::decode(required(meta.get("native_membership")?)?.value())
.map_err(|_| StoreError::CorruptRecord)?;
let last = read_applied(&meta)?;
@@ -458,7 +470,8 @@ impl Store {
let tx = self.write_transaction()?;
{
let mut meta = tx.open_table(META)?;
if !matches!(number(required(meta.get("schema")?)?.value())?, 4 | 6) {
let schema = number(required(meta.get("schema")?)?.value())?;
if !matches!(schema, 4 | 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
if entries.is_empty() {
@@ -471,7 +484,36 @@ impl Store {
if entry.log_id.index != next {
return Err(StoreError::InvalidLogSequence);
}
encoded.push(encode_native_entry(entry)?);
if schema == 7 {
if let openraft::EntryPayload::Membership(membership) = &entry.payload {
// Reuse framing's size preflight before allocating the membership encoding.
let stored = openraft::StoredMembership::new(Some(entry.log_id), membership.clone());
let header = crate::snapshot::Header {
cluster_id: crate::decode_identity(required(meta.get("identity")?)?.value())?.cluster_id,
applied: Some(entry.log_id.into()),
membership: crate::CaptureMembership::try_from(&stored)?,
binding: read_binding(&meta)?,
};
crate::snapshot::Encoder::new(
std::io::sink(),
&header,
[0; 4],
crate::snapshot::Limits {
total_bytes: 1024 * 1024,
membership_bytes: 1024 * 1024 - 256,
key_bytes: 0,
value_bytes: 0,
records: 0,
},
)
.map_err(|_| StoreError::ResourceLimit)?;
}
}
let bytes = encode_native_entry(entry)?;
if schema == 7 && bytes.len() > 1024 * 1024 {
return Err(StoreError::ResourceLimit);
}
encoded.push(bytes);
if position + 1 < entries.len() {
next = next_index(Some(next))?;
}
@@ -487,7 +529,7 @@ impl Store {
}
pub fn read_native_entry(&self, index: u64) -> Result<Option<crate::CaptureRaftEntry>, StoreError> {
let tx = self.database.begin_read()?;
if !matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6) {
if !matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
tx.open_table(LOGS)?
@@ -504,14 +546,14 @@ impl Store {
/// Read typed native entries within the requested bounds.
pub fn read_native_range<R: std::ops::RangeBounds<u64>>(&self, range: R) -> Result<Vec<crate::CaptureRaftEntry>, StoreError> {
let tx = self.database.begin_read()?;
if !matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6) {
if !matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
read_native_range_in(&tx, range)
}
pub fn read_native_entries(&self, start: u64, end: u64) -> Result<Vec<crate::CaptureRaftEntry>, StoreError> {
let tx = self.database.begin_read()?;
if !matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6) {
if !matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
if start > end {
@@ -526,7 +568,7 @@ impl Store {
{
let mut meta = tx.open_table(META)?;
let schema = number(required(meta.get("schema")?)?.value())?;
if !matches!(schema, 4 | 6) {
if !matches!(schema, 4 | 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
let binding = read_binding(&meta)?;
@@ -574,7 +616,7 @@ impl Store {
}
pub fn native_apply_response(&self, index: u64) -> Result<Option<crate::CaptureResponse>, StoreError> {
let tx = self.database.begin_read()?;
if !matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6) {
if !matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
let results = tx.open_table(APPLY_RESULTS)?;
@@ -583,6 +625,19 @@ impl Store {
pub fn native_application_state(&self) -> Result<(Option<LogId>, crate::CaptureMembership), StoreError> {
native_application_state(&self.database.begin_read()?)
}
/// Check that a recovered native state supports provenance-backed snapshots.
/// This does not promote legacy records or change durable metadata.
pub fn validate_native_provenance_state(&self) -> Result<(), StoreError> {
let tx = self.database.begin_read()?;
let schema = number(required(tx.open_table(META)?.get("schema")?)?.value())?;
if !matches!(schema, 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
native_application_state(&tx)?;
read_binding(&tx.open_table(META)?)?;
Ok(())
}
/// Bootstrap the immutable capture policy. The caller must be the trusted
/// deployment provisioning owner. This crate does not authenticate callers;
/// do not expose this public method through an ordinary object request API.
@@ -592,7 +647,7 @@ impl Store {
let tx = self.write_transaction()?;
{
let mut meta = tx.open_table(META)?;
if matches!(number(required(meta.get("schema")?)?.value())?, 3..=6) {
if matches!(number(required(meta.get("schema")?)?.value())?, 3..=7) {
if read_binding(&meta)? == binding {
return Ok(());
}
@@ -609,6 +664,7 @@ impl Store {
if !tx.open_table(APPLY_RESULTS)?.is_empty()? {
return Err(StoreError::CorruptRecord);
}
tx.open_table(crate::source_page::EVENT_SEQUENCE)?.retain(|_, _| false)?;
meta.insert("capture_binding", encoded.as_slice())?;
meta.insert("last_applied", &[0][..])?;
meta.insert("schema", 3u64.to_be_bytes().as_slice())?;
@@ -659,7 +715,7 @@ impl Store {
pub fn capture_binding(&self) -> Result<Option<CaptureBinding>, StoreError> {
let tx = self.database.begin_read()?;
let meta = tx.open_table(META)?;
if !matches!(number(required(meta.get("schema")?)?.value())?, 3..=6) {
if !matches!(number(required(meta.get("schema")?)?.value())?, 3..=7) {
return Ok(None);
}
Ok(Some(read_binding(&meta)?))
@@ -667,7 +723,7 @@ impl Store {
pub fn last_applied(&self) -> Result<Option<LogId>, StoreError> {
let tx = self.database.begin_read()?;
let meta = tx.open_table(META)?;
if !matches!(number(required(meta.get("schema")?)?.value())?, 3..=6) {
if !matches!(number(required(meta.get("schema")?)?.value())?, 3..=7) {
return Ok(None);
}
read_applied(&meta)
@@ -700,7 +756,7 @@ impl Store {
pub fn created_operation_provenance(&self, operation: [u8; 16]) -> Result<Option<CreatedOperationProvenance>, StoreError> {
let tx = self.database.begin_read()?;
let schema = number(required(tx.open_table(META)?.get("schema")?)?.value())?;
if !matches!(schema, 5 | 6) {
if !matches!(schema, 5 | 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
tx.open_table(OPERATIONS)?
@@ -715,6 +771,9 @@ impl Store {
{
let mut meta = tx.open_table(META)?;
let schema = number(required(meta.get("schema")?)?.value())?;
if schema == 7 {
return Ok(());
}
if !matches!(schema, 3..=6) {
return Err(StoreError::UnsupportedSchema);
}
@@ -752,7 +811,7 @@ impl Store {
let mut operations = tx.open_table(OPERATIONS)?;
let results = tx.open_table(APPLY_RESULTS)?;
let first = validate_application_tables(&meta, &logs, &heads, &versions, &events, &operations, &results, true)?;
if matches!(schema, 5 | 6) {
if matches!(schema, 5 | 6 | 7) {
return Ok(());
}
// Keep one encoded row at a time; the replay map contains only first LogIds.
@@ -776,7 +835,7 @@ impl Store {
pub fn created_apply_result(&self, index: u64) -> Result<Option<CreatedApplyResult>, StoreError> {
let tx = self.database.begin_read()?;
let t = tx.open_table(APPLY_RESULTS)?;
if matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6) {
if matches!(number(required(tx.open_table(META)?.get("schema")?)?.value())?, 4 | 6 | 7) {
t.get(index)?
.map(|v| {
decode_response(v.value()).map(|r| {
@@ -796,6 +855,9 @@ impl Store {
pub(crate) fn validate_application(tx: &ReadTransaction) -> Result<(), StoreError> {
let meta = tx.open_table(META)?;
if number(required(meta.get("schema")?)?.value())? == 7 {
return crate::checkpoint_baseline::validate_recovery(tx);
}
if !matches!(number(required(meta.get("schema")?)?.value())?, 3..=6) {
return Ok(());
}
@@ -1080,3 +1142,85 @@ fn validate_application_tables(
}
Ok(first_operations)
}
impl Store {
pub(crate) fn authorize_created(
&self,
expected: &DecideCreated,
decision: LogId,
) -> Result<crate::CommittedCreatedAuthorization, StoreError> {
if !self.writable {
return Err(StoreError::WriteDisabled);
}
expected.encode()?;
let tx = self.database.begin_read()?;
let meta = tx.open_table(META)?;
let schema = number(required(meta.get("schema")?)?.value())?;
if !matches!(schema, 6 | 7) {
return Err(StoreError::UnsupportedSchema);
}
let (applied, _) = native_application_state(&tx)?;
let committed = crate::committed::validate_committed(&meta, &tx.open_table(LOGS)?)?;
let identity = crate::decode_identity(required(meta.get("identity")?)?.value())?;
let binding = read_binding(&meta)?;
let operations = tx.open_table(OPERATIONS)?;
let record = operations
.get(expected.operation_id.as_slice())?
.ok_or(StoreError::CreatedAuthorizationMismatch)?;
let provenance = decode_provenance(record.value())?;
let command = DecideCreated::decode(&provenance.operation.canonical_command)?;
if command.encode()? != provenance.operation.canonical_command {
return Err(StoreError::CorruptRecord);
}
if command.operation_id != expected.operation_id
|| command.object != expected.object
|| command.expected_head != expected.expected_head
|| command.binding_revision != expected.binding_revision
{
return Err(StoreError::CreatedAuthorizationMismatch);
}
if command.prepared != expected.prepared {
return Err(StoreError::CreatedAuthorizationMismatch);
}
let CreatedResult::Created {
version_id,
event_id,
decision_log_id,
} = provenance.operation.result
else {
return Err(StoreError::CreatedAuthorizationMismatch);
};
if decision_log_id != decision
|| provenance.first_decision_log_id != decision
|| version_id != command.prepared.version_id
|| event_id != command.operation_id
|| binding.revision != command.binding_revision
{
return Err(StoreError::CreatedAuthorizationMismatch);
}
for boundary in [applied, committed] {
if boundary.is_none_or(|boundary| {
boundary.index < decision.index || (boundary.index == decision.index && boundary != decision)
}) {
return Err(StoreError::CreatedAuthorizationMismatch);
}
}
let events = tx.open_table(EVENTS)?;
let event = decode_event(required(events.get(command.operation_id.as_slice())?)?.value())?;
let versions = tx.open_table(VERSIONS)?;
let version = decode_version(
required(versions.get(version_key(&command.object, command.prepared.version_id)?.as_slice())?)?.value(),
)?;
if event.object != command.object
|| event.prepared != command.prepared
|| event.decision_log_id != decision
|| event.binding != binding
|| version.prepared != command.prepared
|| version.decision_log_id != decision
|| version.binding_revision != binding.revision
{
return Err(StoreError::CorruptRecord);
}
Ok(crate::CommittedCreatedAuthorization::verified(identity, command, binding, decision))
}
}
@@ -0,0 +1,579 @@
//! Immutable checkpoint state and disk-backed validation of the retained suffix.
use crate::{
CaptureMembership, CreatedApplyResult, CreatedEvent, CreatedResult, CreatedVersion, LogId, StoreError, application,
created::{DecideCreated, Decoder, object_bytes, result_bytes},
native_checkpoint::{CheckpointLimits, recovery_scratch},
snapshot,
};
use redb::{ReadTransaction, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition};
pub(crate) const META_KEY: &str = "checkpoint_baseline";
pub(crate) const PURGED_KEY: &str = "checkpoint_purged";
pub(crate) const HEADS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_heads_v1");
pub(crate) const VERSIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_versions_v1");
pub(crate) const EVENTS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_events_v1");
pub(crate) const OPERATIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_operations_v1");
const FIRST_DECISIONS: TableDefinition<u64, &[u8]> = TableDefinition::new("checkpoint_validation_decisions");
const VERSION_OPERATIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_validation_versions");
const METADATA_BYTES: usize = 1024 * 1024;
#[derive(Clone, Debug)]
pub(crate) struct BaselineMetadata {
pub(crate) generation: u64,
pub(crate) header: snapshot::Header,
pub(crate) limits: CheckpointLimits,
}
struct EmptySink;
impl snapshot::StagingSink for EmptySink {
fn row(&mut self, _: snapshot::Section, _: &[u8], _: &[u8]) -> std::io::Result<()> {
Err(std::io::Error::other("checkpoint metadata contains rows"))
}
}
fn metadata_limits() -> snapshot::Limits {
snapshot::Limits {
total_bytes: 1024 * 1024,
membership_bytes: 512 * 1024,
key_bytes: 0,
value_bytes: 0,
records: 0,
}
}
pub(crate) fn encode_metadata(metadata: &BaselineMetadata) -> Result<Vec<u8>, StoreError> {
metadata.limits.validate()?;
if metadata.generation == 0 {
return Err(StoreError::CorruptRecord);
}
let mut bytes = b"CBAS\x01".to_vec();
for value in [
metadata.generation,
metadata.limits.snapshot.total_bytes,
metadata.limits.snapshot.membership_bytes,
metadata.limits.snapshot.key_bytes,
metadata.limits.snapshot.value_bytes,
metadata.limits.snapshot.records,
metadata.limits.staging_bytes,
] {
bytes.extend(value.to_be_bytes());
}
let frame = snapshot::Encoder::new(Vec::new(), &metadata.header, [0; 4], metadata_limits())
.and_then(snapshot::Encoder::finish)
.map_err(|_| StoreError::CorruptRecord)?;
bytes.extend(frame);
if bytes.len() > METADATA_BYTES {
return Err(StoreError::CorruptRecord);
}
Ok(bytes)
}
pub(crate) fn read_metadata(
meta: &impl ReadableTable<&'static str, &'static [u8]>,
) -> Result<Option<BaselineMetadata>, StoreError> {
let schema = crate::number(required(meta.get("schema")?)?.value())?;
let stored = meta.get(META_KEY)?;
if schema != 7 {
if stored.is_some() || meta.get(PURGED_KEY)?.is_some() {
return Err(StoreError::CorruptRecord);
}
return Ok(None);
}
let value = required(stored)?;
let bytes = value.value();
if bytes.len() > METADATA_BYTES || bytes.get(..5) != Some(b"CBAS\x01") {
return Err(StoreError::CorruptRecord);
}
let mut numbers = [0u64; 7];
for (index, number) in numbers.iter_mut().enumerate() {
let start = 5 + index * 8;
*number = crate::number(bytes.get(start..start + 8).ok_or(StoreError::CorruptRecord)?)?;
}
let limits = CheckpointLimits {
snapshot: snapshot::Limits {
total_bytes: numbers[1],
membership_bytes: numbers[2],
key_bytes: numbers[3],
value_bytes: numbers[4],
records: numbers[5],
},
staging_bytes: numbers[6],
};
limits.validate()?;
let header = snapshot::decode(bytes.get(61..).ok_or(StoreError::CorruptRecord)?, &mut EmptySink, metadata_limits())
.map_err(|_| StoreError::CorruptRecord)?;
let metadata = BaselineMetadata {
generation: numbers[0],
header,
limits,
};
if encode_metadata(&metadata)?.as_slice() != bytes {
return Err(StoreError::CorruptRecord);
}
Ok(Some(metadata))
}
pub(crate) fn read_baseline_cut(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
Ok(read_metadata(meta)?.and_then(|metadata| metadata.header.applied))
}
pub(crate) fn encode_purged(id: LogId) -> Vec<u8> {
let mut bytes = b"CPUR\x01".to_vec();
bytes.extend(crate::encode_id(id));
bytes
}
pub(crate) fn read_purged(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
let cut = read_baseline_cut(meta)?;
let Some(value) = meta.get(PURGED_KEY)? else {
return Ok(None);
};
let bytes = value.value();
if bytes.len() != 29 || bytes.get(..5) != Some(b"CPUR\x01") {
return Err(StoreError::CorruptRecord);
}
let id = crate::decode_id(&bytes[5..])?;
at_or_before(id, required(cut)?)?;
Ok(Some(id))
}
fn at_or_before(id: LogId, cut: LogId) -> Result<(), StoreError> {
if id.index > cut.index || (id.index == cut.index && id != cut) {
return Err(StoreError::CorruptRecord);
}
Ok(())
}
fn required<T>(value: Option<T>) -> Result<T, StoreError> {
value.ok_or(StoreError::CorruptRecord)
}
struct Budget {
limits: snapshot::Limits,
bytes: u64,
records: u64,
}
impl Budget {
fn new(limits: snapshot::Limits) -> Self {
Self {
limits,
bytes: 0,
records: 0,
}
}
fn row(&mut self, key: &[u8], value: &[u8]) -> Result<(), StoreError> {
let key = u64::try_from(key.len()).map_err(|_| StoreError::CorruptRecord)?;
let value = u64::try_from(value.len()).map_err(|_| StoreError::CorruptRecord)?;
if key > 1024 * 1024 || value > 1024 * 1024 {
return Err(StoreError::CorruptRecord);
}
self.bytes = self
.bytes
.checked_add(key)
.and_then(|n| n.checked_add(value))
.ok_or(StoreError::ResourceLimit)?;
self.records = self.records.checked_add(1).ok_or(StoreError::ResourceLimit)?;
if key > self.limits.key_bytes
|| value > self.limits.value_bytes
|| self.bytes > self.limits.total_bytes
|| self.records > self.limits.records
{
return Err(StoreError::ResourceLimit);
}
Ok(())
}
}
pub(crate) fn resource_error(error: StoreError) -> StoreError {
use std::error::Error;
let mut cause: Option<&(dyn Error + 'static)> = Some(&error);
while let Some(current) = cause {
if current
.downcast_ref::<std::io::Error>()
.is_some_and(|io| io.kind() == std::io::ErrorKind::StorageFull)
{
return StoreError::ResourceLimit;
}
cause = current.source();
}
error
}
/// Validate relationships, preserving negative history rather than re-evaluating it.
/// The caller separately establishes local capture or authenticated installation authority.
pub(crate) fn validate_baseline_with_budget(
tx: &ReadTransaction,
metadata: &BaselineMetadata,
index_max: u64,
) -> Result<(), StoreError> {
validate_baseline_inner(tx, metadata, index_max).map_err(resource_error)
}
fn validate_baseline_inner(tx: &ReadTransaction, metadata: &BaselineMetadata, index_max: u64) -> Result<(), StoreError> {
encode_metadata(metadata)?;
let Some(cut) = metadata.header.applied else {
if metadata.header.membership != CaptureMembership::new(None, vec![], std::collections::BTreeMap::new())? {
return Err(StoreError::CorruptRecord);
}
for definition in [HEADS, VERSIONS, EVENTS, OPERATIONS] {
if tx.open_table(definition)?.len()? != 0 {
return Err(StoreError::CorruptRecord);
}
}
return Ok(());
};
let heads = tx.open_table(HEADS)?;
let versions = tx.open_table(VERSIONS)?;
let events = tx.open_table(EVENTS)?;
let operations = tx.open_table(OPERATIONS)?;
let (_directory, index) = recovery_scratch(index_max)?;
let mut index_write = index.begin_write()?;
index_write.open_table(FIRST_DECISIONS)?;
index_write.open_table(VERSION_OPERATIONS)?;
let mut budget = Budget::new(metadata.limits.snapshot);
let mut pending = 0;
for row in operations.iter()? {
let (key, value) = row?;
budget.row(key.value(), value.value())?;
let provenance = application::decode_provenance(value.value())?;
let command = DecideCreated::decode(&provenance.operation.canonical_command)?;
let canonical = command.encode()?;
if key.value() != command.operation_id || canonical != provenance.operation.canonical_command {
return Err(StoreError::CorruptRecord);
}
at_or_before(provenance.first_decision_log_id, cut)?;
let mut original = u64::try_from(canonical.len())
.map_err(|_| StoreError::CorruptRecord)?
.to_be_bytes()
.to_vec();
original.extend(&canonical);
original.extend(result_bytes(&provenance.operation.result));
if application::encode_provenance(provenance.first_decision_log_id, &original)?.as_slice() != value.value() {
return Err(StoreError::CorruptRecord);
}
if index_write
.open_table(FIRST_DECISIONS)?
.insert(provenance.first_decision_log_id.index, key.value())?
.is_some()
{
return Err(StoreError::CorruptRecord);
}
if provenance.operation.result == CreatedResult::OperationConflict {
return Err(StoreError::CorruptRecord);
}
if let CreatedResult::Created {
version_id,
event_id,
decision_log_id,
} = provenance.operation.result
{
if version_id != command.prepared.version_id
|| event_id != command.operation_id
|| decision_log_id != provenance.first_decision_log_id
|| command.binding_revision != metadata.header.binding.revision
{
return Err(StoreError::CorruptRecord);
}
let event = application::decode_event(required(events.get(key.value())?)?.value())?;
if event
!= (CreatedEvent {
object: command.object.clone(),
prepared: command.prepared.clone(),
decision_log_id,
binding: metadata.header.binding.clone(),
})
{
return Err(StoreError::CorruptRecord);
}
let version_key = application::version_key(&command.object, version_id)?;
let version = application::decode_version(required(versions.get(version_key.as_slice())?)?.value())?;
if version
!= (CreatedVersion {
prepared: command.prepared.clone(),
decision_log_id,
binding_revision: metadata.header.binding.revision,
})
{
return Err(StoreError::CorruptRecord);
}
if index_write
.open_table(VERSION_OPERATIONS)?
.insert(version_key.as_slice(), key.value())?
.is_some()
{
return Err(StoreError::CorruptRecord);
}
} else if events.get(key.value())?.is_some() {
return Err(StoreError::CorruptRecord);
}
pending += 1;
if pending == 128 {
index_write.commit().map_err(StoreError::StagingCommit)?;
index_write = index.begin_write()?;
pending = 0;
}
}
index_write.commit().map_err(StoreError::StagingCommit)?;
let index_read = index.begin_read()?;
let version_operations = index_read.open_table(VERSION_OPERATIONS)?;
if version_operations.len()? != versions.len()? {
return Err(StoreError::CorruptRecord);
}
for row in versions.iter()? {
let (key, value) = row?;
budget.row(key.value(), value.value())?;
required(version_operations.get(key.value())?)?;
let mut decoder = Decoder::new(key.value());
let object = decoder.object()?;
let id = decoder.uuid()?;
decoder.finish()?;
let version = application::decode_version(value.value())?;
let head_bytes = required(heads.get(object_bytes(&object)?.as_slice())?)?;
let mut decoder = Decoder::new(head_bytes.value());
let head = decoder.uuid()?;
decoder.finish()?;
let current =
application::decode_version(required(versions.get(application::version_key(&object, head)?.as_slice())?)?.value())?;
if id != version.prepared.version_id
|| current.decision_log_id.index < version.decision_log_id.index
|| (current.decision_log_id.index == version.decision_log_id.index
&& current.decision_log_id != version.decision_log_id)
{
return Err(StoreError::CorruptRecord);
}
}
for row in events.iter()? {
let (key, value) = row?;
budget.row(key.value(), value.value())?;
let event = application::decode_event(value.value())?;
let operation = application::decode_provenance(required(operations.get(key.value())?)?.value())?;
if operation.operation.result
!= (CreatedResult::Created {
version_id: event.prepared.version_id,
event_id: key.value().try_into().map_err(|_| StoreError::CorruptRecord)?,
decision_log_id: event.decision_log_id,
})
{
return Err(StoreError::CorruptRecord);
}
}
for row in heads.iter()? {
let (key, value) = row?;
budget.row(key.value(), value.value())?;
let mut decoder = Decoder::new(key.value());
let object = decoder.object()?;
decoder.finish()?;
let mut decoder = Decoder::new(value.value());
let id = decoder.uuid()?;
decoder.finish()?;
required(versions.get(application::version_key(&object, id)?.as_slice())?)?;
}
Ok(())
}
pub(crate) fn validate_recovery(tx: &ReadTransaction) -> Result<(), StoreError> {
validate_recovery_with_limits(tx, None)
}
pub(crate) fn validate_recovery_with_limits(
tx: &ReadTransaction,
override_limits: Option<CheckpointLimits>,
) -> Result<(), StoreError> {
validate_recovery_inner(tx, override_limits).map_err(resource_error)
}
fn validate_recovery_inner(tx: &ReadTransaction, override_limits: Option<CheckpointLimits>) -> Result<(), StoreError> {
let meta = tx.open_table(crate::META)?;
let metadata = required(read_metadata(&meta)?)?;
crate::raft_state_machine::selected_meta(&meta, &metadata)?;
let cut = metadata.header.applied;
let recovery_limits = override_limits.unwrap_or(metadata.limits);
recovery_limits.validate()?;
let identity = crate::decode_identity(required(meta.get("identity")?)?.value())?;
let binding = application::read_binding(&meta)?;
if identity.cluster_id != metadata.header.cluster_id || binding != metadata.header.binding {
return Err(StoreError::CorruptRecord);
}
let purged = read_purged(&meta)?;
validate_baseline_with_budget(tx, &metadata, recovery_limits.staging_bytes)?;
let last = application::read_applied(&meta)?;
optional_at_or_before(cut, last)?;
let logs = tx.open_table(crate::LOGS)?;
let committed = crate::committed::validate_committed(&meta, &logs)?;
optional_at_or_before(last, committed)?;
let declared_tail = meta.get("tail")?.map(|v| crate::decode_id(v.value())).transpose()?;
optional_at_or_before(committed, declared_tail)?;
let (_directory, scratch) = recovery_scratch(recovery_limits.staging_bytes)?;
for (source, target) in [
(HEADS, application::HEADS),
(VERSIONS, application::VERSIONS),
(EVENTS, application::EVENTS),
(OPERATIONS, application::OPERATIONS),
] {
let table = tx.open_table(source)?;
let mut write = scratch.begin_write()?;
write.open_table(target)?;
let mut pending = 0;
for row in table.iter()? {
let (key, value) = row?;
write.open_table(target)?.insert(key.value(), value.value())?;
pending += 1;
if pending == 128 {
write.commit().map_err(StoreError::StagingCommit)?;
write = scratch.begin_write()?;
pending = 0;
}
}
write.commit().map_err(StoreError::StagingCommit)?;
}
let mut membership: CaptureMembership = metadata.header.membership.clone();
if let Some(cut) = cut
&& let Some(retained_cut) = logs.get(cut.index)?
&& crate::decode_log(retained_cut.value())?.id != cut
{
return Err(StoreError::CorruptRecord);
}
let results = tx.open_table(application::APPLY_RESULTS)?;
// Covered entries may remain until purge. Check their exact typed responses
// without rebuilding the historical state or allocating a prefix-sized map.
let operations = tx.open_table(OPERATIONS)?;
let mut actual_tail = purged;
let mut prefix_next = purged.map_or(Some(0), |id| id.index.checked_add(1));
if let Some(cut) = cut {
for row in logs.range(..=cut.index)? {
let (key, bytes) = row?;
if purged.is_some_and(|purged| key.value() <= purged.index) {
return Err(StoreError::CorruptRecord);
}
if bytes.value().len() > 1024 * 1024 {
return Err(StoreError::CorruptRecord);
}
let entry = crate::decode_log(bytes.value())?;
if entry.id.index != key.value() || prefix_next != Some(key.value()) {
return Err(StoreError::CorruptRecord);
}
actual_tail = Some(entry.id);
prefix_next = key.value().checked_add(1);
let native = application::decode_native_entry(entry)?;
let bytes = required(results.get(key.value())?)?;
if bytes.value().len() > 1024 * 1024 {
return Err(StoreError::CorruptRecord);
}
let response = application::decode_response(bytes.value())?;
application::validate_response_kind(&native, &response)?;
if let openraft::EntryPayload::Normal(command) = native.payload {
let operation = application::decode_provenance(
required(operations.get(command.created().operation_id.as_slice())?)?.value(),
)?;
let canonical = command.created().encode()?;
let expected = if operation.operation.canonical_command == canonical {
operation.operation.result
} else {
CreatedResult::OperationConflict
};
if response.created_result() != Some(&expected) {
return Err(StoreError::CorruptRecord);
}
if operation.first_decision_log_id.index == key.value()
&& operation.first_decision_log_id != LogId::from(native.log_id)
{
return Err(StoreError::CorruptRecord);
}
}
}
for row in results.range(..=cut.index)? {
let (key, _) = row?;
if purged.is_some_and(|purged| key.value() <= purged.index) || logs.get(key.value())?.is_none() {
return Err(StoreError::CorruptRecord);
}
}
if purged.is_none_or(|purged| purged.index < cut.index) && logs.get(cut.index)?.is_none() {
return Err(StoreError::CorruptRecord);
}
}
if actual_tail != cut {
return Err(StoreError::CorruptRecord);
}
let mut observed = cut;
let mut next = cut.map_or(Some(0), |cut| cut.index.checked_add(1));
let mut budget = Budget::new(recovery_limits.snapshot);
if let Some(start) = next {
for row in logs.range(start..)? {
let (key, bytes) = row?;
budget.row(&key.value().to_be_bytes(), bytes.value())?;
let entry = crate::decode_log(bytes.value())?;
if next != Some(key.value()) || entry.id.index != key.value() {
return Err(StoreError::CorruptRecord);
}
next = key.value().checked_add(1);
actual_tail = Some(entry.id);
let native = application::decode_native_entry(entry.clone())?;
if last.is_none_or(|last| key.value() > last.index) {
continue;
}
let response_bytes = required(results.get(key.value())?)?;
if response_bytes.value().len() > 1024 * 1024 {
return Err(StoreError::CorruptRecord);
}
if u64::try_from(response_bytes.value().len()).map_err(|_| StoreError::CorruptRecord)?
> recovery_limits.snapshot.value_bytes
{
return Err(StoreError::ResourceLimit);
}
let response = application::decode_response(response_bytes.value())?;
application::validate_response_kind(&native, &response)?;
let write = scratch.begin_write()?;
let expected = match native.payload {
openraft::EntryPayload::Blank => crate::CaptureResponse::blank(entry.id)?,
openraft::EntryPayload::Membership(value) => {
membership = CaptureMembership::try_from(&openraft::StoredMembership::new(Some(entry.id.into()), value))?;
crate::CaptureResponse::membership(entry.id)?
}
openraft::EntryPayload::Normal(command) => {
let canonical = command.created().encode()?;
let result = application::apply_created(&write, entry.id, command.created(), &canonical, &binding, 6)?;
crate::CaptureResponse::created(CreatedApplyResult {
log_id: entry.id,
result,
})?
}
};
if response != expected {
return Err(StoreError::CorruptRecord);
}
write.commit().map_err(StoreError::StagingCommit)?;
observed = Some(entry.id);
}
for row in results.range(start..)? {
let (key, _) = row?;
if last.is_none_or(|last| key.value() > last.index) || logs.get(key.value())?.is_none() {
return Err(StoreError::CorruptRecord);
}
}
}
if actual_tail != declared_tail || observed != last || application::native_application_state(tx)?.1 != membership {
return Err(StoreError::CorruptRecord);
}
let recovered = scratch.begin_read()?;
let mut live_budget = Budget::new(recovery_limits.snapshot);
for definition in [
application::HEADS,
application::VERSIONS,
application::EVENTS,
application::OPERATIONS,
] {
let expected = recovered.open_table(definition)?;
let actual = tx.open_table(definition)?;
if expected.len()? != actual.len()? {
return Err(StoreError::CorruptRecord);
}
for row in actual.iter()? {
let (key, value) = row?;
live_budget.row(key.value(), value.value())?;
if required(expected.get(key.value())?)?.value() != value.value() {
return Err(StoreError::CorruptRecord);
}
}
}
Ok(())
}
fn optional_at_or_before(id: Option<LogId>, cut: Option<LogId>) -> Result<(), StoreError> {
match (id, cut) {
(None, _) => Ok(()),
(Some(id), Some(cut)) => at_or_before(id, cut),
_ => Err(StoreError::CorruptRecord),
}
}
+11 -10
View File
@@ -8,17 +8,18 @@ pub(crate) fn validate_committed(
let schema = number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
let marker = meta.get("committed")?;
match (schema, marker) {
(1 | 3 | 4 | 5 | 6, None) => {
if matches!(schema, 3..=6) && meta.get("last_applied")?.is_none_or(|v| v.value() != [0]) {
(1 | 3 | 4 | 5 | 6 | 7, None) => {
if matches!(schema, 3..=7) && meta.get("last_applied")?.is_none_or(|v| v.value() != [0]) {
return Err(StoreError::CorruptRecord);
}
Ok(None)
}
(2 | 3 | 4 | 5 | 6, Some(marker)) => {
(2 | 3 | 4 | 5 | 6 | 7, Some(marker)) => {
let id = decode_id(marker.value())?;
let record = logs.get(id.index)?.ok_or(StoreError::CorruptRecord)?;
if decode_log(record.value())?.id != id {
return Err(StoreError::CorruptRecord);
match logs.get(id.index)? {
Some(record) if decode_log(record.value())?.id == id => {}
None if schema == 7 && crate::checkpoint_baseline::read_baseline_cut(meta)? == Some(id) => {}
_ => return Err(StoreError::CorruptRecord),
}
Ok(Some(id))
}
@@ -41,6 +42,9 @@ impl Store {
let mut meta = tx.open_table(META)?;
let logs = tx.open_table(LOGS)?;
let previous = validate_committed(&meta, &logs)?;
if previous == Some(id) {
return Ok(());
}
let record = logs.get(id.index)?.ok_or(StoreError::InvalidCommittedLog)?;
if decode_log(record.value())?.id != id {
return Err(StoreError::InvalidCommittedLog);
@@ -49,12 +53,9 @@ impl Store {
if id.index < previous.index {
return Err(StoreError::CommittedRegression);
}
if id == previous {
return Ok(());
}
}
meta.insert("committed", encode_id(id).as_slice())?;
if previous.is_none() && !matches!(number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?, 3..=6) {
if previous.is_none() && !matches!(number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?, 3..=7) {
meta.insert("schema", 2u64.to_be_bytes().as_slice())?;
}
}
@@ -0,0 +1,278 @@
//! Reuse the existing actual Raft owner fixture, never a caller-constructed result proof.
use super::*;
async fn decide(cluster: &Cluster, value: u8) -> (DecideCreated, LogId) {
let input = command(value);
let expected = input.created().clone();
let response = cluster
.node(0)
.owner()
.client_write(input)
.await
.expect("actual Raft Created decision");
let Some(CreatedResult::Created { decision_log_id, .. }) = response.data.created_result() else {
panic!("successful Created result required")
};
(expected, *decision_log_id)
}
#[tokio::test]
async fn committed_created_authorization_exact() {
let cluster = Cluster::start(&[0], 100).await;
let (expected, first) = decide(&cluster, 1).await;
let authorization = cluster
.node(0)
.owner()
.authorize_created(expected.clone(), first)
.await
.expect("durable exact authorization");
assert_eq!(authorization.identity(), identity(0));
assert_eq!(authorization.operation_id(), expected.operation_id);
assert_eq!(authorization.object(), &expected.object);
assert_eq!(authorization.prepared(), &expected.prepared);
assert_eq!(authorization.binding(), &binding());
assert_eq!(authorization.decision_log_id(), first);
assert_eq!(cluster.node(0).page().await.events[0].captured.decision_log_id, first);
cluster.stop().await;
}
#[tokio::test]
async fn committed_created_authorization_retry_first_decision() {
let cluster = Cluster::start(&[0], 100).await;
let (expected, first) = decide(&cluster, 2).await;
let before = cluster
.node(0)
.owner()
.authorize_created(expected.clone(), first)
.await
.expect("first authorization");
let response = cluster
.node(0)
.owner()
.client_write(CaptureCommand::try_created(expected.clone()).expect("retry command"))
.await
.expect("real retry");
assert_ne!(LogId::from(response.log_id), first);
assert!(
matches!(response.data.created_result(), Some(CreatedResult::Created{decision_log_id,..}) if *decision_log_id==first)
);
let after = cluster
.node(0)
.owner()
.authorize_created(expected.clone(), first)
.await
.expect("first decision retained");
assert_eq!(before, after);
assert!(
cluster
.node(0)
.owner()
.authorize_created(expected, response.log_id.into())
.await
.is_err()
);
assert_eq!(cluster.node(0).page().await.events.len(), 1);
cluster.stop().await;
}
#[tokio::test]
async fn committed_created_authorization_rejects_mismatch() {
let cluster = Cluster::start(&[0], 100).await;
let (expected, first) = decide(&cluster, 3).await;
let before = cluster.node(0).page().await;
let durable_before = cluster
.node(0)
.writer
.write(|store| {
Ok((
store.read_vote()?,
store.read_committed()?,
store.read_entries(0, u64::MAX)?,
store.native_application_state()?,
store.created_events_page(None, 256)?,
))
})
.await
.expect("durable state before rejected authorization");
let mut variants = Vec::new();
let mut altered = expected.clone();
altered.operation_id = [4; 16];
variants.push(altered);
let mut altered = expected.clone();
altered.object.key.push(9);
variants.push(altered);
let mut altered = expected.clone();
altered.object.bucket_incarnation = [9; 16];
variants.push(altered);
let mut altered = expected.clone();
altered.expected_head = Some([9; 16]);
variants.push(altered);
let mut altered = expected.clone();
altered.prepared.preparation_id = [9; 16];
variants.push(altered);
let mut altered = expected.clone();
altered.prepared.version_id = [9; 16];
variants.push(altered);
let mut altered = expected.clone();
altered.prepared.content_digest = [9; 32];
variants.push(altered);
let mut altered = expected.clone();
altered.prepared.content_length += 1;
variants.push(altered);
let mut altered = expected.clone();
altered.binding_revision += 1;
variants.push(altered);
for altered in variants {
assert!(
cluster.node(0).owner().authorize_created(altered, first).await.is_err(),
"AUTHORIZATION_REQUIRES_EXACT_PREPARED_IDENTITY_AND_COMMAND"
);
}
for wrong in [
LogId {
term: first.term + 1,
..first
},
LogId {
leader_node: first.leader_node + 1,
..first
},
LogId {
index: first.index + 1,
..first
},
] {
assert!(
cluster
.node(0)
.owner()
.authorize_created(expected.clone(), wrong)
.await
.is_err()
);
}
let durable_after = cluster
.node(0)
.writer
.write(|store| {
Ok((
store.read_vote()?,
store.read_committed()?,
store.read_entries(0, u64::MAX)?,
store.native_application_state()?,
store.created_events_page(None, 256)?,
))
})
.await
.expect("durable state after rejected authorization");
assert_eq!(durable_before, durable_after, "REJECTED_AUTHORIZATION_DOES_NOT_WRITE");
let mut failed = expected.clone();
failed.operation_id = [5; 16];
failed.prepared.version_id = [5; 16];
let response = cluster
.node(0)
.owner()
.client_write(CaptureCommand::try_created(failed.clone()).expect("failed command"))
.await
.expect("real head mismatch decision");
assert!(matches!(response.data.created_result(), Some(CreatedResult::HeadMismatch { .. })));
assert!(
cluster
.node(0)
.owner()
.authorize_created(failed, response.log_id.into())
.await
.is_err()
);
assert_eq!(cluster.node(0).page().await, before);
cluster.stop().await;
}
#[tokio::test]
async fn committed_created_authorization_restart() {
let routes = Arc::new(Routes::default());
let mut node = Node::start(0, routes.clone(), config(100)).await;
node.owner().bootstrap(members(&[0])).await.expect("bootstrap actual owner");
node.owner()
.raft()
.wait(Some(Duration::from_secs(15)))
.state(ServerState::Leader, "leader")
.await
.expect("leader ready");
let expected = command(6).created().clone();
let response = node.owner().client_write(command(6)).await.expect("real first write");
let Some(CreatedResult::Created {
decision_log_id: first, ..
}) = response.data.created_result()
else {
panic!("Created")
};
let first = *first;
let before = node
.owner()
.authorize_created(expected.clone(), first)
.await
.expect("before restart");
node.stop(&routes).await;
let Node {
directory,
writer,
owner: _,
id,
} = node;
drop(writer);
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity(id)).expect("retained redb"));
// A retained commit-before-apply crash fixture is not a client-provided proof.
// Only real owner initialization may replay it before authorization can succeed.
use openraft::storage::{RaftLogStorage, RaftLogStorageExt};
let pending = command(7).created().clone();
let mut logs = NativeRaftLogStore::new(writer.clone());
let last = logs.get_log_state().await.expect("retained tail").last_log_id.expect("tail");
let pending_id = LogId {
term: last.leader_id.term,
leader_node: last.leader_id.node_id,
index: last.index + 1,
};
logs.blocking_append(vec![CaptureRaftEntry {
log_id: pending_id.into(),
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(pending.clone()).expect("pending command")),
}])
.await
.expect("retained crash suffix");
logs.save_committed(Some(pending_id.into()))
.await
.expect("retained commit marker");
assert!(
writer.authorize_created(pending.clone(), pending_id).await.is_err(),
"UNAPPLIED_COMMIT_IS_NOT_AUTHORIZATION"
);
drop(logs);
let owner = NativeRaftOwner::start(
writer.clone(),
id,
config(100),
CheckpointLimits::default(),
DirectFactory {
source: id,
routes: routes.clone(),
},
)
.await
.expect("real recovery");
routes.register(id, owner.raft().clone());
let after = owner
.authorize_created(expected, first)
.await
.expect("durable authorization after restart");
assert_eq!(before, after);
let replayed = owner
.authorize_created(pending.clone(), pending_id)
.await
.expect("actual owner committed recovery authorizes");
assert_eq!(replayed.prepared(), &pending.prepared);
assert_eq!(replayed.decision_log_id(), pending_id);
owner.shutdown().await.expect("drain recovered owner");
routes.remove(id);
drop(owner);
drop(writer);
}
+54 -4
View File
@@ -1,12 +1,14 @@
use crate::{LogId, StoreError, decode_id, encode_id};
pub const MAX_OBJECT_KEY_LENGTH: usize = 4096;
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ObjectIdentity {
pub bucket_incarnation: [u8; 16],
pub key: Vec<u8>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PreparedIdentity {
pub version_id: [u8; 16],
pub preparation_id: [u8; 16],
@@ -21,7 +23,8 @@ pub struct DecideCreated {
pub prepared: PreparedIdentity,
pub binding_revision: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CaptureBinding {
pub revision: u64,
pub binding_id: [u8; 16],
@@ -47,7 +50,8 @@ pub struct CreatedVersion {
pub decision_log_id: LogId,
pub binding_revision: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CreatedEvent {
pub object: ObjectIdentity,
pub prepared: PreparedIdentity,
@@ -70,6 +74,52 @@ pub struct CreatedApplyResult {
pub result: CreatedResult,
}
/// Local publication authority re-read from the actual owner's durable Created decision.
/// This is not a serializable remote proof and does not attest physical preparation durability.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CommittedCreatedAuthorization {
identity: crate::NodeIdentity,
command: DecideCreated,
binding: CaptureBinding,
decision_log_id: LogId,
}
impl CommittedCreatedAuthorization {
pub(crate) fn verified(
identity: crate::NodeIdentity,
command: DecideCreated,
binding: CaptureBinding,
decision_log_id: LogId,
) -> Self {
Self {
identity,
command,
binding,
decision_log_id,
}
}
pub fn identity(&self) -> crate::NodeIdentity {
self.identity
}
pub fn operation_id(&self) -> [u8; 16] {
self.command.operation_id
}
pub fn object(&self) -> &ObjectIdentity {
&self.command.object
}
pub fn expected_head(&self) -> Option<[u8; 16]> {
self.command.expected_head
}
pub fn prepared(&self) -> &PreparedIdentity {
&self.command.prepared
}
pub fn binding(&self) -> &CaptureBinding {
&self.binding
}
pub fn decision_log_id(&self) -> LogId {
self.decision_log_id
}
}
pub(crate) struct Decoder<'a> {
bytes: &'a [u8],
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,588 @@
//! Local checkpoints are captured from this writer's owned transaction only.
use crate::{
LogId, NativeLogWriter, NativeSnapshotError, NativeSnapshotView, StoreError,
checkpoint_baseline::{self, BaselineMetadata},
snapshot,
};
use redb::{ReadableDatabase, ReadableTable, StorageBackend};
use std::{
fs::File,
io::{self, Seek, Write},
ops::Bound,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
#[derive(Clone, Copy, Debug)]
pub struct CheckpointLimits {
pub snapshot: snapshot::Limits,
pub staging_bytes: u64,
}
impl CheckpointLimits {
pub(crate) fn validate(&self) -> Result<(), StoreError> {
let rows = self.snapshot;
if rows.total_bytes == 0
|| rows.membership_bytes == 0
|| rows.membership_bytes > 512 * 1024
|| rows.key_bytes == 0
|| rows.key_bytes > 1024 * 1024
|| rows.value_bytes == 0
|| rows.value_bytes > 1024 * 1024
|| rows.records == 0
|| self.staging_bytes < 4 * 1024 * 1024
{
return Err(StoreError::ResourceLimit);
}
Ok(())
}
}
impl Default for CheckpointLimits {
fn default() -> Self {
Self {
snapshot: snapshot::Limits {
total_bytes: 256 * 1024 * 1024,
membership_bytes: 512 * 1024,
key_bytes: 8192,
value_bytes: 8192,
records: 1_000_000,
},
staging_bytes: 1024 * 1024 * 1024,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CheckpointReceipt {
generation: u64,
cut: LogId,
}
impl CheckpointReceipt {
pub fn generation(&self) -> u64 {
self.generation
}
pub fn cut(&self) -> LogId {
self.cut
}
}
#[derive(Debug, thiserror::Error)]
pub enum CheckpointError {
#[error(transparent)]
Store(#[from] StoreError),
#[error(transparent)]
Capture(#[from] NativeSnapshotError),
#[error(transparent)]
Snapshot(#[from] snapshot::SnapshotError),
#[error("checkpoint filesystem failure")]
Io(#[from] io::Error),
#[error("checkpoint worker failed")]
Worker(#[source] tokio::task::JoinError),
#[error("checkpoint store lock poisoned")]
Poisoned,
#[error("checkpoint conflict: {0}")]
Conflict(&'static str),
#[error("checkpoint cancelled before publication")]
Cancelled,
}
/// Enforces the file-length budget before redb can allocate or write a page.
#[derive(Debug)]
pub(crate) struct BoundedBackend {
inner: redb::backends::FileBackend,
maximum: u64,
}
impl BoundedBackend {
fn new(file: File, maximum: u64) -> Result<Self, redb::DatabaseError> {
Ok(Self {
inner: redb::backends::FileBackend::new(file)?,
maximum,
})
}
fn check(&self, end: u64) -> io::Result<()> {
if end > self.maximum {
return Err(io::Error::new(io::ErrorKind::StorageFull, "checkpoint staging budget"));
}
Ok(())
}
}
impl StorageBackend for BoundedBackend {
fn len(&self) -> io::Result<u64> {
self.inner.len()
}
fn read(&self, offset: u64, out: &mut [u8]) -> io::Result<()> {
self.inner.read(offset, out)
}
fn set_len(&self, len: u64) -> io::Result<()> {
self.check(len)?;
self.inner.set_len(len)
}
fn sync_data(&self) -> io::Result<()> {
self.inner.sync_data()
}
fn write(&self, offset: u64, data: &[u8]) -> io::Result<()> {
let length = u64::try_from(data.len()).map_err(io::Error::other)?;
self.check(
offset
.checked_add(length)
.ok_or_else(|| io::Error::other("checkpoint write overflow"))?,
)?;
self.inner.write(offset, data)
}
fn close(&self) -> io::Result<()> {
self.inner.close()
}
fn try_lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
self.inner.try_lock_range(start, end)
}
fn try_lock_shared_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
self.inner.try_lock_shared_range(start, end)
}
fn lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
self.inner.lock_range(start, end)
}
fn lock_shared_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
self.inner.lock_shared_range(start, end)
}
fn unlock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
self.inner.unlock_range(start, end)
}
fn query_lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
self.inner.query_lock_range(start, end)
}
}
pub(crate) fn recovery_scratch(maximum: u64) -> Result<(tempfile::TempDir, redb::Database), StoreError> {
let directory = tempfile::tempdir()?;
let database = stage_database(directory.path().join("stage.redb"), maximum)?;
Ok((directory, database))
}
pub(crate) fn stage_database(path: std::path::PathBuf, maximum: u64) -> Result<redb::Database, StoreError> {
let file = File::options().read(true).write(true).create_new(true).open(path)?;
let backend = BoundedBackend::new(file, maximum)?;
Ok(redb::Database::builder()
.set_cache_size(8 * 1024 * 1024)
.create_with_backend(backend)?)
}
struct PayloadFile {
file: File,
maximum: u64,
written: u64,
}
impl Write for PayloadFile {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
let length = u64::try_from(bytes.len()).map_err(io::Error::other)?;
let end = self
.written
.checked_add(length)
.ok_or_else(|| io::Error::other("checkpoint payload overflow"))?;
if end > self.maximum {
return Err(io::Error::new(io::ErrorKind::StorageFull, "checkpoint payload budget"));
}
let count = self.file.write(bytes)?;
self.written = self
.written
.checked_add(u64::try_from(count).map_err(io::Error::other)?)
.ok_or_else(|| io::Error::other("checkpoint payload overflow"))?;
Ok(count)
}
fn flush(&mut self) -> io::Result<()> {
self.file.flush()
}
}
pub(crate) struct StageSink<'a>(pub(crate) &'a redb::WriteTransaction);
impl snapshot::StagingSink for StageSink<'_> {
fn row(&mut self, section: snapshot::Section, key: &[u8], value: &[u8]) -> io::Result<()> {
let definition = match section {
snapshot::Section::Heads => checkpoint_baseline::HEADS,
snapshot::Section::Versions => checkpoint_baseline::VERSIONS,
snapshot::Section::Events => checkpoint_baseline::EVENTS,
snapshot::Section::Operations => checkpoint_baseline::OPERATIONS,
};
let mut table = self.0.open_table(definition).map_err(io::Error::other)?;
if table.insert(key, value).map_err(io::Error::other)?.is_some() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "duplicate checkpoint key"));
}
Ok(())
}
}
// This type has no external constructors, deserialization, or path-based import.
// Its source remains the owned local read transaction, not the decoded artifact.
struct LocalStage {
source: NativeSnapshotView,
database: redb::Database,
_directory: tempfile::TempDir,
metadata: BaselineMetadata,
}
fn capture_stage(source: NativeSnapshotView, limits: CheckpointLimits) -> Result<LocalStage, CheckpointError> {
limits.validate()?;
source
.header()
.applied
.ok_or(CheckpointError::Conflict("empty applied cut"))?;
let directory = tempfile::tempdir()?;
let payload = File::options()
.read(true)
.write(true)
.create_new(true)
.open(directory.path().join("capture.csnp"))?;
let artifact = source.stream_local_capability(
PayloadFile {
file: payload,
maximum: limits.staging_bytes.min(limits.snapshot.total_bytes),
written: 0,
},
limits.snapshot,
)?;
let remaining = limits
.staging_bytes
.checked_sub(artifact.payload.written)
.ok_or(CheckpointError::Conflict("staging budget"))?;
let mut payload = artifact.payload.file;
payload.sync_all()?;
payload.rewind()?;
let database = stage_database(directory.path().join("stage.redb"), remaining)?;
let transaction = database.begin_write().map_err(StoreError::from)?;
for definition in [
checkpoint_baseline::HEADS,
checkpoint_baseline::VERSIONS,
checkpoint_baseline::EVENTS,
checkpoint_baseline::OPERATIONS,
] {
transaction.open_table(definition).map_err(StoreError::from)?;
}
let header = snapshot::decode_capability_artifact(
snapshot::CapabilityArtifact {
manifest: artifact.manifest,
payload,
},
&mut StageSink(&transaction),
limits.snapshot,
)?;
std::fs::remove_file(directory.path().join("capture.csnp"))?;
if header != *source.header() {
return Err(CheckpointError::Conflict("capture header"));
}
let metadata = BaselineMetadata {
generation: 1,
header,
limits,
};
let encoded = checkpoint_baseline::encode_metadata(&metadata)?;
{
let mut meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
meta.insert("schema", 7u64.to_be_bytes().as_slice())
.map_err(StoreError::from)?;
meta.insert("checkpoint_baseline", encoded.as_slice())
.map_err(StoreError::from)?;
}
transaction.commit().map_err(StoreError::StagingCommit)?;
let stage = LocalStage {
source,
database,
_directory: directory,
metadata,
};
Ok(stage)
}
fn verify_local_stage(stage: &LocalStage) -> Result<(), CheckpointError> {
let transaction = stage.database.begin_read().map_err(StoreError::from)?;
for (source_definition, stage_definition) in [
(crate::application::HEADS, checkpoint_baseline::HEADS),
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
] {
let source = stage
.source
.transaction()
.open_table(source_definition)
.map_err(StoreError::from)?;
let staged = transaction.open_table(stage_definition).map_err(StoreError::from)?;
let mut source_rows = source.iter().map_err(StoreError::from)?;
let mut staged_rows = staged.iter().map_err(StoreError::from)?;
loop {
match (source_rows.next(), staged_rows.next()) {
(None, None) => break,
(Some(source), Some(staged)) => {
let (source_key, source_value) = source.map_err(StoreError::from)?;
let (stage_key, stage_value) = staged.map_err(StoreError::from)?;
if source_key.value() != stage_key.value() || source_value.value() != stage_value.value() {
return Err(CheckpointError::Conflict("staging differs from owned local capture"));
}
}
_ => return Err(CheckpointError::Conflict("staging row count")),
}
}
}
let staged_meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
let staged_metadata =
checkpoint_baseline::read_metadata(&staged_meta)?.ok_or(CheckpointError::Conflict("missing staging metadata"))?;
if checkpoint_baseline::encode_metadata(&staged_metadata)? != checkpoint_baseline::encode_metadata(&stage.metadata)? {
return Err(CheckpointError::Conflict("staging metadata differs from local capture"));
}
let stage_bytes = std::fs::metadata(stage._directory.path().join("stage.redb"))?.len();
let index_budget = stage
.metadata
.limits
.staging_bytes
.checked_sub(stage_bytes)
.ok_or(CheckpointError::Conflict("staging disk budget"))?;
checkpoint_baseline::validate_baseline_with_budget(&transaction, &stage.metadata, index_budget)?;
Ok(())
}
fn resource_error(error: CheckpointError) -> CheckpointError {
use std::error::Error;
if matches!(&error, CheckpointError::Store(StoreError::CommitIndeterminate(_))) {
return error;
}
let mut source: Option<&(dyn Error + 'static)> = Some(&error);
while let Some(current) = source {
if current
.downcast_ref::<io::Error>()
.is_some_and(|error| error.kind() == io::ErrorKind::StorageFull)
{
return CheckpointError::Store(StoreError::ResourceLimit);
}
source = current.source();
}
error
}
struct CancelOnDrop(Arc<AtomicBool>);
impl Drop for CancelOnDrop {
fn drop(&mut self) {
self.0.store(true, Ordering::Release);
}
}
#[cfg(test)]
type CaptureHook = Box<dyn FnOnce() + Send>;
#[cfg(test)]
type StageHook = Box<dyn FnOnce(&redb::Database) + Send>;
#[cfg(test)]
static CAPTURE_HOOKS: std::sync::Mutex<std::collections::BTreeMap<usize, CaptureHook>> =
std::sync::Mutex::new(std::collections::BTreeMap::new());
#[cfg(test)]
static STAGE_HOOKS: std::sync::Mutex<std::collections::BTreeMap<usize, StageHook>> =
std::sync::Mutex::new(std::collections::BTreeMap::new());
#[cfg(test)]
pub(crate) struct CheckpointHook {
owner: usize,
stage: bool,
}
#[cfg(test)]
impl Drop for CheckpointHook {
fn drop(&mut self) {
if self.stage {
STAGE_HOOKS.lock().expect("stage hooks").remove(&self.owner);
} else {
CAPTURE_HOOKS.lock().expect("capture hooks").remove(&self.owner);
}
}
}
#[cfg(test)]
pub(crate) fn after_local_capture(writer: &NativeLogWriter, hook: impl FnOnce() + Send + 'static) -> CheckpointHook {
let owner = Arc::as_ptr(&writer.store).addr();
assert!(
CAPTURE_HOOKS
.lock()
.expect("capture hooks")
.insert(owner, Box::new(hook))
.is_none()
);
CheckpointHook { owner, stage: false }
}
#[cfg(test)]
pub(crate) fn after_local_stage(writer: &NativeLogWriter, hook: impl FnOnce(&redb::Database) + Send + 'static) -> CheckpointHook {
let owner = Arc::as_ptr(&writer.store).addr();
assert!(
STAGE_HOOKS
.lock()
.expect("stage hooks")
.insert(owner, Box::new(hook))
.is_none()
);
CheckpointHook { owner, stage: true }
}
#[cfg(test)]
fn run_capture_hook(owner: &Arc<std::sync::Mutex<crate::Store>>) {
let hook = CAPTURE_HOOKS
.lock()
.expect("capture hooks")
.remove(&Arc::as_ptr(owner).addr());
if let Some(hook) = hook {
hook();
}
}
#[cfg(test)]
fn run_stage_hook(owner: &Arc<std::sync::Mutex<crate::Store>>, database: &redb::Database) {
let hook = STAGE_HOOKS.lock().expect("stage hooks").remove(&Arc::as_ptr(owner).addr());
if let Some(hook) = hook {
hook(database);
}
}
impl NativeLogWriter {
pub async fn checkpoint_local(&self, limits: CheckpointLimits) -> Result<CheckpointReceipt, CheckpointError> {
self.checkpoint_local_inner(limits).await.map_err(resource_error)
}
async fn checkpoint_local_inner(&self, limits: CheckpointLimits) -> Result<CheckpointReceipt, CheckpointError> {
limits.validate()?;
let admission = self.checkpoint_admission.clone().lock_owned().await;
let owner = self.store.clone();
let stage = tokio::task::spawn_blocking(move || {
let source = NativeSnapshotView::acquire(owner.clone())?;
#[cfg(test)]
run_capture_hook(&owner);
let stage = capture_stage(source, limits)?;
#[cfg(test)]
run_stage_hook(&owner, &stage.database);
verify_local_stage(&stage)?;
Ok::<_, CheckpointError>((stage, admission))
})
.await
.map_err(CheckpointError::Worker)??;
let ordering = self.ordering.clone().lock_owned().await;
let owner = self.store.clone();
let cancellation = CancelOnDrop(Arc::new(AtomicBool::new(false)));
let cancelled = cancellation.0.clone();
tokio::task::spawn_blocking(move || {
// The blocking owner retains both guards until publication finishes,
// even if the async caller is dropped while redb is committing.
let _ordering = ordering;
let (stage, _admission) = stage;
if cancelled.load(Ordering::Acquire) {
return Err(CheckpointError::Cancelled);
}
if !Arc::ptr_eq(&owner, stage.source.owner()) {
return Err(CheckpointError::Conflict("local capture owner"));
}
let mut store = owner.lock().map_err(|_| CheckpointError::Poisoned)?;
let transaction = store.write_transaction()?;
let mut meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
let schema = crate::number(
meta.get("schema")
.map_err(StoreError::from)?
.ok_or(StoreError::CorruptRecord)?
.value(),
)?;
if !matches!(schema, 6 | 7) {
return Err(StoreError::UnsupportedSchema.into());
}
let previous = checkpoint_baseline::read_metadata(&meta)?;
if crate::application::read_binding(&meta)? != stage.metadata.header.binding {
return Err(CheckpointError::Conflict("live capture binding changed"));
}
let identity = crate::decode_identity(
meta.get("identity")
.map_err(StoreError::from)?
.ok_or(StoreError::CorruptRecord)?
.value(),
)?;
if identity.cluster_id != stage.metadata.header.cluster_id {
return Err(CheckpointError::Conflict("live capture cluster changed"));
}
let cut = stage
.metadata
.header
.applied
.ok_or(CheckpointError::Conflict("empty applied cut"))?;
let applied = crate::application::read_applied(&meta)?.ok_or(StoreError::CorruptRecord)?;
let logs = transaction.open_table(crate::LOGS).map_err(StoreError::from)?;
let committed = crate::committed::validate_committed(&meta, &logs)?.ok_or(StoreError::CorruptRecord)?;
if cut.index > applied.index
|| (cut.index == applied.index && cut != applied)
|| applied.index > committed.index
|| (applied.index == committed.index && applied != committed)
{
return Err(CheckpointError::Conflict("live applied and committed bounds"));
}
drop(logs);
if let Some(old) = previous.as_ref().and_then(|previous| previous.header.applied)
&& (old.index > cut.index || (old.index == cut.index && old != cut))
{
return Err(CheckpointError::Conflict("published checkpoint cut"));
}
let mut metadata = stage.metadata;
metadata.generation = previous.map_or(Ok(1), |old| {
old.generation
.checked_add(1)
.ok_or(CheckpointError::Conflict("checkpoint generation exhausted"))
})?;
let stage_transaction = stage.database.begin_read().map_err(StoreError::from)?;
for (source_definition, definition) in [
(crate::application::HEADS, checkpoint_baseline::HEADS),
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
] {
let captured = stage
.source
.transaction()
.open_table(source_definition)
.map_err(StoreError::from)?;
let staged = stage_transaction.open_table(definition).map_err(StoreError::from)?;
let mut source_rows = captured.iter().map_err(StoreError::from)?;
let mut stage_rows = staged.iter().map_err(StoreError::from)?;
let mut target = transaction.open_table(definition).map_err(StoreError::from)?;
target.retain(|_, _| false).map_err(StoreError::from)?;
// Recheck disk bytes while copying; the earlier relationship
// validation remains outside the writer ordering interval.
loop {
match (source_rows.next(), stage_rows.next()) {
(None, None) => break,
(Some(source), Some(staged)) => {
let (source_key, source_value) = source.map_err(StoreError::from)?;
let (key, value) = staged.map_err(StoreError::from)?;
if source_key.value() != key.value() || source_value.value() != value.value() {
return Err(CheckpointError::Conflict("publication differs from local capture"));
}
target.insert(key.value(), value.value()).map_err(StoreError::from)?;
}
_ => return Err(CheckpointError::Conflict("publication row count")),
}
}
}
let encoded = checkpoint_baseline::encode_metadata(&metadata)?;
meta.insert("checkpoint_baseline", encoded.as_slice())
.map_err(StoreError::from)?;
meta.insert("schema", 7u64.to_be_bytes().as_slice())
.map_err(StoreError::from)?;
crate::raft_state_machine::publish_local_meta(&mut meta, &metadata)?;
drop(meta);
if cancelled.load(Ordering::Acquire) {
return Err(CheckpointError::Cancelled);
}
store.commit(transaction)?;
Ok(CheckpointReceipt {
generation: metadata.generation,
cut,
})
})
.await
.map_err(CheckpointError::Worker)?
}
}
@@ -0,0 +1,607 @@
use super::*;
use std::collections::{BTreeMap, BTreeSet};
fn identity() -> NodeIdentity {
NodeIdentity {
cluster_id: [7; 16],
node_id: 0,
}
}
fn binding() -> CaptureBinding {
CaptureBinding {
revision: 9,
binding_id: [8; 16],
target_id: [9; 16],
}
}
fn id(index: u64) -> LogId {
LogId {
term: 8,
leader_node: 0,
index,
}
}
fn command(operation: u8) -> DecideCreated {
DecideCreated {
operation_id: [operation; 16],
object: ObjectIdentity {
bucket_incarnation: [2; 16],
key: b"checkpoint-object".to_vec(),
},
expected_head: None,
prepared: PreparedIdentity {
version_id: [3; 16],
preparation_id: [4; 16],
content_digest: [5; 32],
content_length: 4097,
},
binding_revision: 9,
}
}
fn apply(store: &mut Store, index: u64, command: DecideCreated) -> CaptureResponse {
let entry = CaptureRaftEntry {
log_id: id(index).into(),
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("canonical command")),
};
store.append_native(&[entry]).expect("actual durable append");
store.save_committed(id(index)).expect("actual durable commit");
store.apply_next_native(id(index)).expect("actual native apply")
}
fn fixture() -> (tempfile::TempDir, Store, DecideCreated, CreatedOperationProvenance) {
let dir = tempfile::tempdir().expect("actual native database directory");
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize actual redb");
store.pin_capture_binding(binding()).expect("bind actual store");
store.enable_native_empty().expect("native schema4");
store.promote_created_provenance().expect("native schema6 provenance");
let membership = CaptureMembership::new(
Some(id(0)),
vec![BTreeSet::from([0])],
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("native member"))]),
)
.expect("full native membership");
let stored = openraft::StoredMembership::try_from(&membership).expect("OpenRaft membership");
store
.append_native(&[CaptureRaftEntry {
log_id: id(0).into(),
payload: openraft::EntryPayload::Membership(stored.membership().clone()),
}])
.expect("append actual membership");
store.save_committed(id(0)).expect("commit membership");
store.apply_next_native(id(0)).expect("apply membership");
let negative = command(11);
let mut original = negative.clone();
original.expected_head = Some([12; 16]);
apply(&mut store, 1, original.clone());
let first = store
.created_operation_provenance(original.operation_id)
.expect("read first negative")
.expect("canonical negative operation");
assert_eq!(first.first_decision_log_id, id(1), "CHECKPOINT_FIRST_NEGATIVE_BOUND");
assert!(
matches!(first.operation.result, CreatedResult::HeadMismatch { .. }),
"CHECKPOINT_FIRST_NEGATIVE_BOUND"
);
apply(&mut store, 2, command(1));
(dir, store, original, first)
}
fn runtime() -> tokio::runtime::Runtime {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("checkpoint runtime")
}
fn schema(store: &Store) -> u64 {
let tx = store.database.begin_read().expect("read actual schema");
let table = tx.open_table(META).expect("schema table");
number(table.get("schema").expect("schema read").expect("schema exists").value()).expect("schema number")
}
fn assert_negative(store: &Store, original: &DecideCreated, first: &CreatedOperationProvenance) {
assert_eq!(
store
.created_operation_provenance(original.operation_id)
.expect("canonical read"),
Some(first.clone()),
"CHECKPOINT_FIRST_NEGATIVE_BOUND: preserve original bytes/result/full first ID"
);
}
#[test]
fn native_checkpoint_negative_retry_and_purge_reopen() {
let (dir, store, original, first) = fixture();
assert_eq!(schema(&store), 6);
let writer = NativeLogWriter::new(store);
let receipt = runtime()
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
.expect("publish local checkpoint");
assert_eq!(receipt.cut(), id(2), "CHECKPOINT_BASELINE_BOUND");
assert_eq!(receipt.generation(), 1);
{
let mut store = writer.store.lock().expect("actual writer store");
assert_eq!(schema(&store), 7);
assert_negative(&store, &original, &first);
apply(&mut store, 3, original.clone());
assert_negative(&store, &original, &first);
let mut conflict = original.clone();
conflict.binding_revision += 1;
apply(&mut store, 4, conflict);
assert_eq!(
store
.created_apply_result(4)
.expect("conflicting response")
.expect("response")
.result,
CreatedResult::OperationConflict,
"CHECKPOINT_RETRY_BOUND"
);
assert_negative(&store, &original, &first);
}
runtime()
.block_on(writer.purge_checkpoint(&receipt))
.expect("purge exact durable baseline");
drop(writer);
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen purged baseline plus suffix");
assert_negative(&reopened, &original, &first);
assert_eq!(reopened.last_applied().expect("suffix applied"), Some(id(4)), "CHECKPOINT_SUFFIX_BOUND");
assert!(reopened.read_entry(2).expect("purged cut").is_none());
assert!(reopened.read_entry(3).expect("retained suffix").is_some());
}
#[test]
fn native_checkpoint_capture_keeps_concurrent_suffix() {
let (dir, store, original, first) = fixture();
let writer = NativeLogWriter::new(store);
let (entered_tx, entered_rx) = std::sync::mpsc::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel();
let _hook = native_checkpoint::after_local_capture(&writer, move || {
entered_tx.send(()).expect("capture reached");
release_rx
.recv_timeout(std::time::Duration::from_secs(30))
.expect("release owned capture");
});
let worker = writer.clone();
let thread = std::thread::spawn(move || runtime().block_on(worker.checkpoint_local(CheckpointLimits::default())));
entered_rx
.recv_timeout(std::time::Duration::from_secs(30))
.expect("actual capture boundary");
{
let mut store = writer.store.lock().expect("capture releases live write owner");
apply(&mut store, 3, original.clone());
}
release_tx.send(()).expect("release capture");
let receipt = thread.join().expect("checkpoint worker").expect("concurrent checkpoint");
assert_eq!(receipt.cut(), id(2), "CHECKPOINT_SAME_CUT_BOUND");
runtime()
.block_on(writer.purge_checkpoint(&receipt))
.expect("purge captured prefix");
drop(writer);
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen concurrent suffix");
assert_eq!(reopened.last_applied().expect("actual suffix"), Some(id(3)), "CHECKPOINT_SUFFIX_BOUND");
assert_negative(&reopened, &original, &first);
}
#[test]
fn native_checkpoint_staging_corruption_preserves_live_generation() {
let (dir, store, original, first) = fixture();
let writer = NativeLogWriter::new(store);
let before = runtime()
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
.expect("initial baseline");
let _hook = native_checkpoint::after_local_stage(&writer, |database| {
let tx = database.begin_write().expect("actual isolated stage mutation");
tx.open_table(checkpoint_baseline::OPERATIONS)
.expect("stage operations")
.retain(|_, _| false)
.expect("damage cross-record relationships");
tx.commit().expect("persist actual damaged stage");
});
assert!(
runtime()
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
.is_err(),
"CHECKPOINT_FAILURE_ATOMIC"
);
let store = writer.store.lock().expect("unchanged live store");
let tx = store.database.begin_read().expect("published generation");
let meta = tx.open_table(META).expect("published meta");
assert_eq!(
checkpoint_baseline::read_metadata(&meta)
.expect("valid old metadata")
.expect("old baseline")
.generation,
before.generation(),
"CHECKPOINT_FAILURE_ATOMIC"
);
assert_negative(&store, &original, &first);
drop(meta);
drop(tx);
drop(store);
drop(writer);
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen after failed staging validation");
assert_negative(&reopened, &original, &first);
}
#[test]
fn native_checkpoint_budget_failure_preserves_schema_and_history() {
for staging in [false, true] {
let (dir, mut store, original, first) = fixture();
if staging {
for index in 0u64..256 {
let mut large = command(31);
large.operation_id[..8].copy_from_slice(&index.to_be_bytes());
large.prepared.version_id[..8].copy_from_slice(&index.to_be_bytes());
large.object.key = vec![b'x'; MAX_OBJECT_KEY_LENGTH];
large.object.key[..8].copy_from_slice(&index.to_be_bytes());
apply(&mut store, index + 3, large);
}
}
let writer = NativeLogWriter::new(store);
let mut limits = CheckpointLimits::default();
if staging {
limits.staging_bytes = 4 * 1024 * 1024;
} else {
limits.snapshot.total_bytes = 64;
}
let result = runtime().block_on(writer.checkpoint_local(limits));
if staging {
assert!(
matches!(&result, Err(CheckpointError::Store(StoreError::ResourceLimit))),
"CHECKPOINT_RESOURCE_BOUND: actual 4MiB physical staging limit: {result:?}"
);
} else {
assert!(
result.is_err(),
"CHECKPOINT_RESOURCE_BOUND: actual bounded output must refuse: {result:?}"
);
}
{
let store = writer.store.lock().expect("live after exhausted budget");
assert_eq!(schema(&store), 6, "CHECKPOINT_FAILURE_ATOMIC");
assert_negative(&store, &original, &first);
assert!(store.read_entry(0).expect("prefix intact").is_some());
}
drop(writer);
assert_negative(
&Store::open(dir.path().join("db"), identity()).expect("reopen budget failure"),
&original,
&first,
);
}
}
#[test]
fn native_checkpoint_cancelled_capture_cannot_publish_or_purge() {
let (dir, store, original, first) = fixture();
let writer = NativeLogWriter::new(store);
let (entered_tx, entered_rx) = std::sync::mpsc::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel();
let _hook = native_checkpoint::after_local_capture(&writer, move || {
entered_tx.send(()).expect("capture cancellation boundary");
release_rx
.recv_timeout(std::time::Duration::from_secs(30))
.expect("release pending actual capture");
});
let rt = runtime();
let worker = writer.clone();
rt.block_on(async {
let task = tokio::spawn(async move { worker.checkpoint_local(CheckpointLimits::default()).await });
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(std::time::Duration::from_secs(30)))
.await
.expect("capture observer")
.expect("actual capture owned transaction");
task.abort();
assert!(task.await.expect_err("cancelled waiter").is_cancelled(), "CHECKPOINT_CANCEL_BOUND");
release_tx.send(()).expect("release actual worker after waiter cancellation");
// A fresh checkpoint waits for the same production admission; completion
// establishes that the cancelled capture worker released its real resources.
let receipt = writer
.checkpoint_local(CheckpointLimits::default())
.await
.expect("capture after cancellation");
assert_eq!(receipt.generation(), 1, "CHECKPOINT_CANCEL_BOUND: abandoned capture never published");
});
drop(rt);
drop(writer);
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen after cancellation and successful next capture");
assert_negative(&reopened, &original, &first);
assert!(
reopened.read_entry(0).expect("cancellation cannot purge").is_some(),
"CHECKPOINT_CANCEL_BOUND"
);
}
#[test]
fn native_checkpoint_stale_receipt_cannot_remove_history() {
let (_dir, store, original, first) = fixture();
let writer = NativeLogWriter::new(store);
let rt = runtime();
let old = rt
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
.expect("first generation");
let new = rt
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
.expect("next generation");
assert!(new.generation() > old.generation());
assert!(
matches!(
rt.block_on(writer.purge_checkpoint(&old)),
Err(NativeWriteError::Store(StoreError::StaleCheckpoint))
),
"CHECKPOINT_PURGE_BOUND"
);
let store = writer.store.lock().expect("history after stale purge");
assert_negative(&store, &original, &first);
assert!(store.read_entry(1).expect("first decision retained").is_some());
assert!(store.created_apply_result(1).expect("typed result retained").is_some());
}
#[test]
fn native_checkpoint_reopen_rejects_conflicting_cut_and_suffix_gap() {
for corruption in 0..4 {
let gap = corruption == 1;
let (dir, store, original, _) = fixture();
let writer = NativeLogWriter::new(store);
let rt = runtime();
let receipt = rt
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
.expect("valid checkpoint");
if gap {
{
let mut store = writer.store.lock().expect("suffix owner");
apply(&mut store, 3, original.clone());
apply(&mut store, 4, original.clone());
}
rt.block_on(writer.purge_checkpoint(&receipt))
.expect("purge before gap corruption");
{
let store = writer.store.lock().expect("actual purged metadata owner");
let tx = store.database.begin_write().expect("actual CPUR mutation");
let mut meta = tx.open_table(META).expect("actual CPUR table");
let mut conflict = receipt.cut();
conflict.term += 1;
conflict.leader_node += 1;
let encoded = checkpoint_baseline::encode_purged(conflict);
meta.insert(checkpoint_baseline::PURGED_KEY, encoded.as_slice())
.expect("persist conflicting full purge ID");
drop(meta);
tx.commit().expect("durable conflicting CPUR");
let tx = store.database.begin_read().expect("actual conflicting CPUR read");
let meta = tx.open_table(META).expect("actual persisted CPUR");
assert!(
matches!(checkpoint_baseline::read_purged(&meta), Err(StoreError::CorruptRecord)),
"CHECKPOINT_FULL_CUT_BOUND: same index with different term/leader is corruption"
);
drop(meta);
drop(tx);
let tx = store
.database
.begin_write()
.expect("restore valid purge boundary for gap scenario");
let encoded = checkpoint_baseline::encode_purged(receipt.cut());
tx.open_table(META)
.expect("actual purge meta")
.insert(checkpoint_baseline::PURGED_KEY, encoded.as_slice())
.expect("restore exact cut");
tx.commit().expect("persist exact purge boundary");
}
}
if corruption >= 2 {
let mut store = writer.store.lock().expect("actual suffix before tail corruption");
apply(&mut store, 3, original.clone());
}
{
let store = writer.store.lock().expect("actual corruption owner");
let tx = store.database.begin_write().expect("actual redb mutation");
let mut logs = tx.open_table(LOGS).expect("actual log table");
if gap {
logs.remove(3).expect("remove required suffix entry");
} else if corruption >= 2 {
let mut tail = id(3);
if corruption == 2 {
tail.index += 1;
} else {
tail.term += 1;
tail.leader_node += 1;
}
let encoded = encode_id(tail);
tx.open_table(META)
.expect("actual metadata")
.insert("tail", encoded.as_slice())
.expect("corrupt durable tail");
} else {
let mut entry = store.read_entry(2).expect("retained cut").expect("cut entry");
entry.id.term += 1;
entry.id.leader_node += 1;
let encoded = encode_log(&entry).expect("valid conflicting full ID encoding");
logs.insert(2, encoded.as_slice()).expect("replace exact cut identity");
}
drop(logs);
tx.commit().expect("persist actual inconsistent recovery input");
}
assert!(
rt.block_on(writer.purge_checkpoint(&receipt)).is_err(),
"CHECKPOINT_PURGE_BOUND: invalid recovery cannot authorize cleanup"
);
if !gap {
let store = writer.store.lock().expect("refused purge retains original evidence");
assert!(store.read_entry(1).expect("original log retained").is_some(), "CHECKPOINT_PURGE_BOUND");
assert!(
store.created_apply_result(1).expect("original response retained").is_some(),
"CHECKPOINT_PURGE_BOUND"
);
}
drop(writer);
drop(rt);
assert!(Store::open(dir.path().join("db"), identity()).is_err(), "CHECKPOINT_RECOVERY_BOUND");
}
}
#[test]
fn native_checkpoint_publication_fault_selects_complete_generation() {
for (sync, after) in [(1, false), (1, true), (2, false), (2, true)] {
let (dir, store, original, first) = fixture();
let vote = VoteRecord {
term: 9,
voted_node: 0,
committed: true,
};
let mut store = store;
store.save_vote(vote).expect("local hard state before checkpoint");
drop(store);
let path = dir.path().join("db");
let backend = tests::CrashBackend::new(&path);
let store = Store::from_database(backend.database(), identity()).expect("existing simulated durability backend");
let writer = NativeLogWriter::new(store);
let (entered, release) = backend.pause_sync(sync, after);
let worker = writer.clone();
let task = std::thread::spawn(move || runtime().block_on(worker.checkpoint_local(CheckpointLimits::default())));
entered.wait();
backend.cut();
release.wait();
let _unknown = task.join().expect("interrupted publication worker");
drop(writer);
let reopened = Store::open(&path, identity()).expect("recover complete generation after simulated sync cut");
assert!(matches!(schema(&reopened), 6 | 7), "CHECKPOINT_CRASH_ATOMIC");
assert_negative(&reopened, &original, &first);
assert_eq!(
reopened.read_vote().expect("preserved local vote"),
Some(vote),
"CHECKPOINT_LOCAL_STATE_BOUND"
);
assert_eq!(reopened.last_applied().expect("complete applied position"), Some(id(2)));
}
}
#[test]
fn native_checkpoint_recovery_budget_can_be_raised_without_changing_baseline() {
use redb::ReadableTableMetadata;
let (dir, store, original, first) = fixture();
let mut limits = CheckpointLimits::default();
let tx = store.database.begin_read().expect("measure real baseline rows");
limits.snapshot.records = [
application::HEADS,
application::VERSIONS,
application::EVENTS,
application::OPERATIONS,
]
.into_iter()
.map(|definition| {
tx.open_table(definition)
.expect("actual baseline table")
.len()
.expect("actual row count")
})
.sum();
drop(tx);
let writer = NativeLogWriter::new(store);
let receipt = runtime()
.block_on(writer.checkpoint_local(limits))
.expect("capture exact bounded baseline");
{
let mut store = writer.store.lock().expect("actual suffix writer");
let mut suffix = command(21);
suffix.object.key = b"suffix-new-key".to_vec();
suffix.prepared.version_id = [22; 16];
apply(&mut store, 3, suffix);
}
drop(writer);
let path = dir.path().join("db");
assert!(
matches!(Store::open(&path, identity()), Err(StoreError::ResourceLimit)),
"CHECKPOINT_RESOURCE_BOUND"
);
let mut raised = limits;
raised.snapshot.records += 32;
let reopened =
Store::open_with_checkpoint_limits(&path, identity(), raised).expect("explicitly raise bounded recovery budget");
assert_negative(&reopened, &original, &first);
assert_eq!(reopened.last_applied().expect("retained suffix position"), Some(id(3)));
let tx = reopened.database.begin_read().expect("unchanged baseline metadata");
let meta = tx.open_table(META).expect("actual metadata");
let baseline = checkpoint_baseline::read_metadata(&meta)
.expect("baseline decode")
.expect("baseline selected");
assert_eq!(baseline.generation, receipt.generation());
assert_eq!(
baseline.limits.snapshot.records, limits.snapshot.records,
"CHECKPOINT_RESOURCE_BOUND: override never rewrites evidence"
);
}
#[test]
fn native_checkpoint_interrupted_purge_reopens_and_retries_idempotently() {
for (sync, after) in [(1, false), (1, true), (2, false), (2, true)] {
let (dir, store, original, first) = fixture();
let writer = NativeLogWriter::new(store);
let receipt = runtime()
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
.expect("durable baseline before purge");
drop(writer);
let path = dir.path().join("db");
let backend = tests::CrashBackend::new(&path);
let store = Store::from_database(backend.database(), identity()).expect("simulated purge durability backend");
let writer = NativeLogWriter::new(store);
let (entered, release) = backend.pause_sync(sync, after);
let worker = writer.clone();
let task = std::thread::spawn(move || runtime().block_on(worker.purge_checkpoint(&receipt)));
entered.wait();
backend.cut();
release.wait();
let _unknown = task.join().expect("interrupted purge worker");
drop(writer);
let reopened = Store::open(&path, identity()).expect("baseline survives simulated interrupted purge");
assert_negative(&reopened, &original, &first);
let retained = (0..=2)
.map(|index| reopened.read_entry(index).expect("atomic purge log state").is_some())
.collect::<Vec<_>>();
assert!(
retained.iter().all(|present| *present) || retained.iter().all(|present| !*present),
"CHECKPOINT_PURGE_ATOMIC: complete old prefix or complete new purge"
);
let writer = NativeLogWriter::new(reopened);
let rt = runtime();
rt.block_on(writer.purge_checkpoint(&receipt)).expect("resume exact purge");
rt.block_on(writer.purge_checkpoint(&receipt))
.expect("repeat exact purge idempotently");
drop(writer);
let reopened = Store::open(&path, identity()).expect("reopen resumed purge");
assert_negative(&reopened, &original, &first);
assert!(reopened.read_entry(2).expect("purged cut absent").is_none(), "CHECKPOINT_PURGE_ATOMIC");
}
}
#[test]
fn native_checkpoint_ack_and_purge_ack_survive_simulated_power_cut() {
let (dir, store, original, first) = fixture();
drop(store);
let path = dir.path().join("db");
let backend = tests::CrashBackend::new(&path);
let writer =
NativeLogWriter::new(Store::from_database(backend.database(), identity()).expect("actual checkpoint fault backend"));
let receipt = runtime()
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
.expect("successful checkpoint ACK");
backend.cut();
drop(writer);
let reopened = Store::open(&path, identity()).expect("ACK checkpoint survives simulated power cut");
assert_eq!(schema(&reopened), 7, "CHECKPOINT_ACK_DURABLE");
assert_negative(&reopened, &original, &first);
let backend = tests::CrashBackend::new(&path);
drop(reopened);
let writer = NativeLogWriter::new(Store::from_database(backend.database(), identity()).expect("actual purge fault backend"));
runtime()
.block_on(writer.purge_checkpoint(&receipt))
.expect("successful purge ACK");
backend.cut();
drop(writer);
let reopened = Store::open(&path, identity()).expect("ACK purge survives simulated power cut");
assert_negative(&reopened, &original, &first);
assert!(reopened.read_entry(2).expect("ACK purge cut absent").is_none(), "CHECKPOINT_ACK_DURABLE");
}
@@ -6,6 +6,30 @@ use std::{
sync::{Arc, Mutex},
};
#[cfg(test)]
std::thread_local! {
static AFTER_CAPTURE: std::cell::RefCell<Option<Box<dyn FnOnce()>>> = const { std::cell::RefCell::new(None) };
}
#[cfg(test)]
pub(crate) struct CaptureHook;
#[cfg(test)]
impl Drop for CaptureHook {
fn drop(&mut self) {
AFTER_CAPTURE.with(|hook| {
hook.borrow_mut().take();
});
}
}
#[cfg(test)]
pub(crate) fn after_capture(hook: impl FnOnce() + 'static) -> CaptureHook {
AFTER_CAPTURE.with(|slot| {
assert!(slot.borrow().is_none(), "one capture hook per test thread");
*slot.borrow_mut() = Some(Box::new(hook));
});
CaptureHook
}
#[derive(Debug, thiserror::Error)]
pub enum NativeSnapshotError {
#[error(transparent)]
@@ -23,6 +47,7 @@ pub struct NativeSnapshotView {
transaction: ReadTransaction,
header: snapshot::Header,
counts: [u64; 4],
schema: u64,
_owner: Arc<Mutex<Store>>,
}
impl NativeSnapshotView {
@@ -34,19 +59,40 @@ impl NativeSnapshotView {
}
store.database.begin_read().map_err(StoreError::from)?
};
#[cfg(test)]
if let Some(hook) = AFTER_CAPTURE.with(|slot| slot.borrow_mut().take()) {
hook();
}
let (applied, membership) = application::native_application_state(&transaction)?;
let header = {
let (header, schema) = {
let meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
let logs = transaction.open_table(crate::LOGS).map_err(StoreError::from)?;
let committed = crate::committed::validate_committed(&meta, &logs)?;
if let Some(applied) = applied {
let committed = committed.ok_or(StoreError::CorruptRecord)?;
if applied.index > committed.index || (applied.index == committed.index && applied != committed) {
return Err(StoreError::CorruptRecord.into());
}
}
let identity = meta
.get("identity")
.map_err(StoreError::from)?
.ok_or(StoreError::CorruptRecord)?;
snapshot::Header {
cluster_id: crate::decode_identity(identity.value())?.cluster_id,
applied,
membership,
binding: application::read_binding(&meta)?,
}
let schema = crate::number(
meta.get("schema")
.map_err(StoreError::from)?
.ok_or(StoreError::CorruptRecord)?
.value(),
)?;
(
snapshot::Header {
cluster_id: crate::decode_identity(identity.value())?.cluster_id,
applied,
membership,
binding: application::read_binding(&meta)?,
},
schema,
)
};
let mut counts = [0; 4];
for (index, definition) in [
@@ -68,11 +114,52 @@ impl NativeSnapshotView {
transaction,
header,
counts,
schema,
_owner: owner,
})
}
pub fn stream_capability_artifact<W: Write>(
self,
writer: W,
limits: snapshot::Limits,
) -> Result<snapshot::CapabilityArtifact<W>, NativeSnapshotError> {
if !matches!(self.schema, 6 | 7) {
return Err(StoreError::UnsupportedSchema.into());
}
let output = snapshot::CapabilityOutput::new(writer, limits.total_bytes);
Ok(self.stream(output, limits)?.finish())
}
pub(crate) fn transaction(&self) -> &ReadTransaction {
&self.transaction
}
pub(crate) fn header(&self) -> &snapshot::Header {
&self.header
}
pub(crate) fn owner(&self) -> &Arc<Mutex<Store>> {
&self._owner
}
pub(crate) fn stream_local_capability<W: Write>(
&self,
writer: W,
limits: snapshot::Limits,
) -> Result<snapshot::CapabilityArtifact<W>, NativeSnapshotError> {
if !matches!(self.schema, 6 | 7) {
return Err(StoreError::UnsupportedSchema.into());
}
let output = snapshot::CapabilityOutput::new(writer, limits.total_bytes);
Ok(self.stream_borrowed(output, limits)?.finish())
}
pub fn stream<W: Write>(self, writer: W, limits: snapshot::Limits) -> Result<W, NativeSnapshotError> {
self.stream_borrowed(writer, limits)
}
fn stream_borrowed<W: Write>(&self, writer: W, limits: snapshot::Limits) -> Result<W, NativeSnapshotError> {
let mut encoder = snapshot::Encoder::new(writer, &self.header, self.counts, limits)?;
for (definition, section) in [
(application::HEADS, snapshot::Section::Heads),
@@ -0,0 +1,197 @@
//! Bounded snapshot transport bytes; receive ownership follows the buffer into install.
use std::{
io::{self, Read, Seek, SeekFrom, Write},
pin::Pin,
task::{Context, Poll},
};
use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite, ReadBuf};
pub struct NativeSnapshotData {
bytes: Vec<u8>,
position: u64,
limit: u64,
pub(crate) receive_guard: Option<crate::raft_state_machine::ReceiveGuard>,
}
fn boundary() -> io::Error {
io::Error::new(io::ErrorKind::InvalidInput, "native snapshot byte limit")
}
impl NativeSnapshotData {
pub fn new(max_bytes: u64) -> io::Result<Self> {
if max_bytes == 0 || usize::try_from(max_bytes).is_err() {
return Err(boundary());
}
Ok(Self {
bytes: Vec::new(),
position: 0,
limit: max_bytes,
receive_guard: None,
})
}
pub(crate) fn as_slice(&self) -> &[u8] {
&self.bytes
}
pub(crate) fn attach_receive_guard(&mut self, guard: crate::raft_state_machine::ReceiveGuard) {
self.receive_guard = Some(guard);
}
}
impl Read for NativeSnapshotData {
fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
let position = usize::try_from(self.position).map_err(|_| boundary())?;
let available = self.bytes.get(position..).unwrap_or_default();
let count = output.len().min(available.len());
output[..count].copy_from_slice(&available[..count]);
self.position = self
.position
.checked_add(u64::try_from(count).map_err(|_| boundary())?)
.ok_or_else(boundary)?;
Ok(count)
}
}
impl Write for NativeSnapshotData {
fn write(&mut self, input: &[u8]) -> io::Result<usize> {
if input.is_empty() {
return Ok(0);
}
let count = u64::try_from(input.len()).map_err(|_| boundary())?;
let end = self
.position
.checked_add(count)
.filter(|end| *end <= self.limit)
.ok_or_else(boundary)?;
let position = usize::try_from(self.position).map_err(|_| boundary())?;
let length = usize::try_from(end).map_err(|_| boundary())?;
if length > self.bytes.len() {
self.bytes
.try_reserve_exact(length - self.bytes.len())
.map_err(|_| io::Error::new(io::ErrorKind::OutOfMemory, "native snapshot allocation"))?;
self.bytes.resize(length, 0);
}
self.bytes[position..length].copy_from_slice(input);
self.position = end;
Ok(input.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
impl Seek for NativeSnapshotData {
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
let position = match from {
SeekFrom::Start(position) => Some(position),
SeekFrom::Current(offset) => self.position.checked_add_signed(offset),
SeekFrom::End(offset) => u64::try_from(self.bytes.len())
.map_err(|_| boundary())?
.checked_add_signed(offset),
}
.filter(|position| *position <= self.limit)
.ok_or_else(boundary)?;
self.position = position;
Ok(position)
}
}
impl AsyncRead for NativeSnapshotData {
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let this = self.get_mut();
let position = match usize::try_from(this.position) {
Ok(position) => position,
Err(_) => return Poll::Ready(Err(boundary())),
};
let available = this.bytes.get(position..).unwrap_or_default();
let count = output.remaining().min(available.len());
let end = this
.position
.checked_add(u64::try_from(count).map_err(|_| boundary())?)
.ok_or_else(boundary)?;
output.put_slice(&available[..count]);
this.position = end;
Poll::Ready(Ok(()))
}
}
impl AsyncWrite for NativeSnapshotData {
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
Poll::Ready(Write::write(self.get_mut(), input))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
impl AsyncSeek for NativeSnapshotData {
fn start_seek(self: Pin<&mut Self>, from: SeekFrom) -> io::Result<()> {
Seek::seek(self.get_mut(), from).map(|_| ())
}
fn poll_complete(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<u64>> {
Poll::Ready(Ok(self.position))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn native_snapshot_data_rejects_growth_before_mutation() {
let mut data = NativeSnapshotData::new(4).expect("bounded snapshot");
Write::write_all(&mut data, b"ab").expect("initial bytes");
assert!(Write::write_all(&mut data, b"cde").is_err());
assert_eq!(data.as_slice(), b"ab", "rejected write leaves bytes intact");
assert_eq!(Seek::stream_position(&mut data).expect("position"), 2);
Seek::seek(&mut data, SeekFrom::Start(4)).expect("seek exactly to limit");
Write::write(&mut data, b"").expect("empty write");
assert_eq!(data.as_slice(), b"ab", "empty write never allocates a gap");
assert!(Write::write(&mut data, b"x").is_err());
assert!(Seek::seek(&mut data, SeekFrom::Current(1)).is_err());
assert!(Seek::seek(&mut data, SeekFrom::Current(i64::MIN)).is_err());
assert_eq!(Seek::stream_position(&mut data).expect("unchanged position"), 4);
Seek::seek(&mut data, SeekFrom::Start(3)).expect("bounded sparse position");
Write::write_all(&mut data, b"z").expect("write ending exactly at limit");
assert_eq!(data.as_slice(), b"ab\0z");
}
#[test]
fn native_snapshot_data_checks_length_overflow() {
assert!(NativeSnapshotData::new(0).is_err());
if usize::BITS == 64 {
let mut data = NativeSnapshotData::new(u64::MAX).expect("representable byte limit");
Seek::seek(&mut data, SeekFrom::Start(u64::MAX)).expect("seek does not allocate");
assert!(Write::write(&mut data, b"x").is_err());
assert!(Seek::seek(&mut data, SeekFrom::Current(1)).is_err());
assert!(data.as_slice().is_empty(), "overflow never allocates");
}
}
#[tokio::test]
async fn native_snapshot_data_async_io_uses_the_same_bounds() {
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
let mut data = NativeSnapshotData::new(4).expect("bounded async snapshot");
AsyncWriteExt::write_all(&mut data, b"abcd").await.expect("write exact limit");
assert!(AsyncWriteExt::write_all(&mut data, b"e").await.is_err());
assert!(AsyncSeekExt::seek(&mut data, SeekFrom::Start(5)).await.is_err());
AsyncSeekExt::seek(&mut data, SeekFrom::End(-2))
.await
.expect("rewind within snapshot");
let mut output = Vec::new();
AsyncReadExt::read_to_end(&mut data, &mut output)
.await
.expect("read snapshot suffix");
assert_eq!(output, b"cd");
AsyncWriteExt::shutdown(&mut data).await.expect("transport shutdown");
assert_eq!(data.as_slice(), b"abcd");
}
}
@@ -0,0 +1,217 @@
//! Validate isolated received bytes before one durable shared-owner publication.
use crate::{
CaptureMembership, CheckpointLimits, NativeLogWriter, NativeSnapshotData, StoreError, application, checkpoint_baseline,
native_checkpoint, snapshot,
};
use redb::{ReadableDatabase, ReadableTable};
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
struct Cancel(Arc<AtomicBool>);
impl Drop for Cancel {
fn drop(&mut self) {
self.0.store(true, Ordering::Release);
}
}
struct Stage {
database: redb::Database,
_directory: tempfile::TempDir,
baseline: checkpoint_baseline::BaselineMetadata,
}
fn stage(data: &NativeSnapshotData, limits: CheckpointLimits) -> Result<Stage, StoreError> {
let bytes = data.as_slice();
if u64::try_from(bytes.len()).map_err(|_| StoreError::ResourceLimit)?
> limits.snapshot.total_bytes.checked_add(56).ok_or(StoreError::ResourceLimit)?
{
return Err(StoreError::ResourceLimit);
}
let manifest = snapshot::CapabilityManifest::decode(bytes.get(..56).ok_or(StoreError::CorruptRecord)?)
.map_err(|_| StoreError::CorruptRecord)?;
let directory = tempfile::tempdir()?;
let database = native_checkpoint::stage_database(directory.path().join("received.redb"), limits.staging_bytes)?;
let tx = database.begin_write()?;
for definition in [
checkpoint_baseline::HEADS,
checkpoint_baseline::VERSIONS,
checkpoint_baseline::EVENTS,
checkpoint_baseline::OPERATIONS,
] {
tx.open_table(definition)?;
}
let header = snapshot::decode_capability_artifact(
snapshot::CapabilityArtifact {
manifest,
payload: &bytes[56..],
},
&mut native_checkpoint::StageSink(&tx),
limits.snapshot,
)
.map_err(|_| StoreError::CorruptRecord)?;
let baseline = checkpoint_baseline::BaselineMetadata {
generation: 1,
header,
limits,
};
let encoded = checkpoint_baseline::encode_metadata(&baseline)?;
{
let mut meta = tx.open_table(crate::META)?;
meta.insert("schema", 7u64.to_be_bytes().as_slice())?;
meta.insert(checkpoint_baseline::META_KEY, encoded.as_slice())?;
}
tx.commit().map_err(StoreError::StagingCommit)?;
checkpoint_baseline::validate_baseline_with_budget(&database.begin_read()?, &baseline, limits.staging_bytes)?;
Ok(Stage {
database,
_directory: directory,
baseline,
})
}
pub(crate) async fn install(
writer: NativeLogWriter,
limits: CheckpointLimits,
meta: openraft::SnapshotMeta<u64, crate::CaptureNode>,
data: Box<NativeSnapshotData>,
) -> Result<(), StoreError> {
if data
.receive_guard
.as_ref()
.is_none_or(|guard| !guard.belongs_to(&writer.snapshot_receives))
{
return Err(StoreError::CorruptRecord);
}
let (mut stage, data) = tokio::task::spawn_blocking(move || Ok::<_, StoreError>((stage(&data, limits)?, data)))
.await
.map_err(|_| StoreError::CorruptRecord)??;
if meta.last_log_id.map(crate::LogId::from) != stage.baseline.header.applied {
return Err(StoreError::CorruptRecord);
}
if CaptureMembership::try_from(&meta.last_membership)? != stage.baseline.header.membership {
return Err(StoreError::CorruptRecord);
}
let ordering = writer.ordering.clone().lock_owned().await;
let cancel = Cancel(Arc::new(AtomicBool::new(false)));
let cancelled = cancel.0.clone();
tokio::task::spawn_blocking(move || {
let _ordering = ordering;
let _data = data;
if cancelled.load(Ordering::Acquire) {
return Err(StoreError::WriteDisabled);
}
let mut store = writer.store.lock().map_err(|_| StoreError::CorruptRecord)?;
let read = store.database.begin_read()?;
let old_meta = read.open_table(crate::META)?;
let identity = crate::decode_identity(old_meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
if identity.cluster_id != stage.baseline.header.cluster_id
|| application::read_binding(&old_meta)? != stage.baseline.header.binding
{
return Err(StoreError::CorruptRecord);
}
let old_applied = application::read_applied(&old_meta)?;
let old_logs = read.open_table(crate::LOGS)?;
let committed = crate::committed::validate_committed(&old_meta, &old_logs)?;
let cut = stage.baseline.header.applied;
if old_applied.is_some_and(|id| cut.is_none_or(|cut| id.index > cut.index || (id.index == cut.index && id != cut))) {
return Err(StoreError::CommittedRegression);
}
let compatible = match cut {
None => true,
Some(cut) => {
let at_cut = old_logs
.get(cut.index)?
.map(|v| crate::decode_log(v.value()).map(|v| v.id))
.transpose()?
.or(checkpoint_baseline::read_baseline_cut(&old_meta)?.filter(|id| id.index == cut.index));
if committed
.is_some_and(|id| id.index >= cut.index && (at_cut != Some(cut) || (id.index == cut.index && id != cut)))
{
return Err(StoreError::CommittedRegression);
}
at_cut == Some(cut)
}
};
let previous = checkpoint_baseline::read_metadata(&old_meta)?;
stage.baseline.generation =
previous.map_or(Ok(1), |old| old.generation.checked_add(1).ok_or(StoreError::ResourceLimit))?;
drop(old_logs);
drop(old_meta);
drop(read);
let source = stage.database.begin_read()?;
let tx = store.write_transaction()?;
for (baseline, live) in [
(checkpoint_baseline::HEADS, application::HEADS),
(checkpoint_baseline::VERSIONS, application::VERSIONS),
(checkpoint_baseline::EVENTS, application::EVENTS),
(checkpoint_baseline::OPERATIONS, application::OPERATIONS),
] {
let table = source.open_table(baseline)?;
for target in [baseline, live] {
let mut target = tx.open_table(target)?;
target.retain(|_, _| false)?;
for row in table.iter()? {
let (key, value) = row?;
target.insert(key.value(), value.value())?;
}
}
}
{
let mut index = tx.open_table(crate::source_page::EVENT_SEQUENCE)?;
index.retain(|_, _| false)?;
}
for row in source.open_table(checkpoint_baseline::EVENTS)?.iter()? {
let (key, value) = row?;
let event = application::decode_event(value.value())?;
crate::source_page::insert_event_index(
&tx,
event.decision_log_id,
key.value().try_into().map_err(|_| StoreError::CorruptRecord)?,
)?;
}
let tail = {
let mut logs = tx.open_table(crate::LOGS)?;
logs.retain(|index, _| compatible && cut.is_none_or(|cut| index > cut.index))?;
logs.last()?
.map(|(_, v)| crate::decode_log(v.value()).map(|v| v.id))
.transpose()?
.or(cut)
};
tx.open_table(application::APPLY_RESULTS)?.retain(|_, _| false)?;
{
let mut target = tx.open_table(crate::META)?;
target.insert("schema", 7u64.to_be_bytes().as_slice())?;
target.insert(
checkpoint_baseline::META_KEY,
checkpoint_baseline::encode_metadata(&stage.baseline)?.as_slice(),
)?;
let mut applied = vec![0];
if let Some(cut) = cut {
target.insert(checkpoint_baseline::PURGED_KEY, checkpoint_baseline::encode_purged(cut).as_slice())?;
applied = vec![1];
applied.extend(crate::encode_id(cut));
} else {
target.remove(checkpoint_baseline::PURGED_KEY)?;
}
target.insert("last_applied", applied.as_slice())?;
target.insert("native_membership", stage.baseline.header.membership.encode()?.as_slice())?;
if let Some(tail) = tail {
target.insert("tail", crate::encode_id(tail).as_slice())?;
} else {
target.remove("tail")?;
}
let committed = committed.filter(|id| cut.is_none_or(|cut| id.index > cut.index)).or(cut);
if let Some(committed) = committed {
target.insert("committed", crate::encode_id(committed).as_slice())?;
} else {
target.remove("committed")?;
}
crate::raft_state_machine::publish_meta(&mut target, &stage.baseline, meta)?;
}
if cancelled.load(Ordering::Acquire) {
return Err(StoreError::WriteDisabled);
}
store.commit(tx)
})
.await
.map_err(|_| StoreError::CorruptRecord)?
}
@@ -0,0 +1,536 @@
//! Atomic installation failures over real native stores and disk fault backend.
use super::raft_state_machine_tests::{Fixture, entries, entry, id, install, receive, snapshot_bytes};
use super::*;
use openraft::{
RaftLogReader, RaftSnapshotBuilder,
storage::{RaftLogStorage, RaftLogStorageExt, RaftStateMachine},
};
async fn media_target() -> (Fixture, tests::CrashBackend) {
let mut f = Fixture::new(1);
f.apply(entries(1)).await;
f.logs
.save_vote(&openraft::Vote::new_committed(9, 1))
.await
.expect("preserved target vote");
let Fixture {
directory,
writer,
logs,
sm,
identity,
} = f;
drop(sm);
drop(logs);
drop(writer);
let backend = tests::CrashBackend::new(&directory.path().join("db"));
let writer = NativeLogWriter::new(Store::from_database(backend.database(), identity).expect("actual redb fault target"));
let logs = NativeRaftLogStore::new(writer.clone());
let sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("fault target state machine");
(
Fixture {
directory,
writer,
logs,
sm,
identity,
},
backend,
)
}
#[tokio::test]
async fn native_snapshot_install_corruption_preserves_state() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let mut target = Fixture::new(1);
target.apply(entries(1)).await;
let before = target.page().await;
let state = target.sm.applied_state().await.expect("old applied state");
let mut corrupted = bytes.clone();
*corrupted.last_mut().expect("nonempty snapshot") ^= 1;
for payload in [corrupted, bytes[..bytes.len() - 1].to_vec()] {
let stable = std::fs::read(target.directory.path().join("db")).expect("old stable bytes");
let data = receive(&mut target.sm, &payload).await;
assert!(
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
"SNAPSHOT_INSTALL_CORRUPT_FRAME_REJECTED"
);
assert_eq!(target.sm.applied_state().await.expect("unchanged applied state"), state);
assert_eq!(target.page().await, before, "SNAPSHOT_INSTALL_CORRUPTION_NO_PARTIAL_TABLES");
assert_eq!(std::fs::read(target.directory.path().join("db")).expect("unchanged stable bytes"), stable);
assert!(
target
.sm
.get_current_snapshot()
.await
.expect("unpublished corrupt snapshot")
.is_none()
);
}
install(&mut target.sm, &mut snapshot).await;
assert_eq!(target.page().await, source.page().await, "SNAPSHOT_INSTALL_RETRY_AFTER_CORRUPT_RECEIVE");
}
#[tokio::test]
async fn native_snapshot_install_stale_cut_rejected() {
let mut source = Fixture::new(0);
source.apply(entries(1)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let mut target = Fixture::new(1);
target.apply(entries(3)).await;
let current = target.build().await.meta;
let events = target.page().await;
let data = receive(&mut target.sm, &bytes).await;
assert!(
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
"SNAPSHOT_INSTALL_NEVER_ROLLS_BACK_APPLIED_STATE"
);
assert_eq!(target.sm.applied_state().await.expect("non-regressed applied").0, Some(id(3).into()));
assert_eq!(
target
.sm
.get_current_snapshot()
.await
.expect("old current snapshot")
.expect("present")
.meta,
current
);
assert_eq!(target.page().await, events);
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let mut target = Fixture::new(1);
let mut batch = entries(3);
for entry in &mut batch[2..] {
entry.log_id = LogId {
term: 9,
..id(entry.log_id.index)
}
.into();
}
target
.logs
.blocking_append(batch)
.await
.expect("conflicting committed suffix logs");
target.sm.apply(entries(1)).await.expect("existing applied prefix");
target
.logs
.save_committed(Some(LogId { term: 9, ..id(3) }.into()))
.await
.expect("protect conflicting committed suffix");
let before = target
.logs
.get_log_reader()
.await
.try_get_log_entries(..)
.await
.expect("protected logs");
let data = receive(&mut target.sm, &bytes).await;
assert!(
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
"SNAPSHOT_INSTALL_NEVER_DISCARDS_COMMITTED_CONFLICT"
);
assert_eq!(
target
.logs
.get_log_reader()
.await
.try_get_log_entries(..)
.await
.expect("unchanged protected logs"),
before
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn native_snapshot_install_cancelled_owner() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let (mut target, backend) = media_target().await;
let data = receive(&mut target.sm, &bytes).await;
let (entered, resume) = backend.pause_sync(1, false);
let mut installing = target.sm.clone();
let meta = snapshot.meta.clone();
let task = tokio::spawn(async move { installing.install_snapshot(&meta, data).await });
entered.wait();
assert!(!task.is_finished(), "SNAPSHOT_INSTALL_NOT_ACKED_BEFORE_SYNC");
task.abort();
assert!(task.await.expect_err("cancelled install waiter").is_cancelled());
assert!(
target.writer.ordering.try_lock().is_err(),
"SNAPSHOT_INSTALL_CANCELLED_WAITER_RETAINS_OWNER"
);
resume.wait();
// Acquiring the production owner waits for the abandoned blocking commit.
drop(target.writer.ordering.lock().await);
assert_eq!(
target.sm.applied_state().await.expect("completed background install").0,
Some(id(2).into()),
"SNAPSHOT_INSTALL_COMMIT_NOT_INTERRUPTED_BY_WAITER_CANCEL"
);
let vote = target.logs.read_vote().await.expect("preserved vote");
assert_eq!(vote, Some(openraft::Vote::new_committed(9, 1)));
backend.cut();
let Fixture {
directory,
writer,
logs,
sm,
identity,
} = target;
drop(sm);
drop(logs);
drop(writer);
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("reopen cancelled installation"));
let mut sm =
NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopen installed state machine");
assert_eq!(
sm.get_current_snapshot()
.await
.expect("completed durable snapshot")
.expect("present")
.meta,
snapshot.meta
);
assert_eq!(
writer.created_events_page(None, 256).await.expect("installed events"),
source.page().await
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn native_snapshot_install_power_cut_atomic() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let new_events = source.page().await;
for after_sync in [false, true] {
let (mut target, backend) = media_target().await;
let old_events = target.page().await;
let data = receive(&mut target.sm, &bytes).await;
let (entered, resume) = backend.pause_sync(1, after_sync);
let meta = snapshot.meta.clone();
let mut installing = target.sm.clone();
let task = tokio::spawn(async move { installing.install_snapshot(&meta, data).await });
entered.wait();
backend.cut();
resume.wait();
assert!(
task.await.expect("faulted install task").is_err(),
"SNAPSHOT_INSTALL_POWER_CUT_NO_FALSE_ACK"
);
assert!(
target.sm.get_current_snapshot().await.is_err(),
"SNAPSHOT_INSTALL_FENCED_HANDLE_CANNOT_EXPORT"
);
let Fixture {
directory,
writer,
logs,
sm,
identity,
} = target;
drop(sm);
drop(logs);
drop(writer);
let writer =
NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("atomic installation recovery"));
let mut logs = NativeRaftLogStore::new(writer.clone());
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("recovered SM");
let events = writer
.created_events_page(None, 256)
.await
.expect("complete recovered tables");
let cut = sm.applied_state().await.expect("recovered applied marker").0;
if events == old_events {
assert_eq!(cut, Some(id(1).into()), "SNAPSHOT_INSTALL_OLD_TABLES_OLD_MARKER");
assert!(sm.get_current_snapshot().await.expect("old snapshot selection").is_none());
} else {
assert_eq!(events, new_events, "SNAPSHOT_INSTALL_ALL_TABLES_OLD_OR_NEW");
assert_eq!(cut, Some(id(2).into()), "SNAPSHOT_INSTALL_NEW_TABLES_NEW_MARKER");
let mut current = sm
.get_current_snapshot()
.await
.expect("new snapshot selection")
.expect("new snapshot published");
assert_eq!(current.meta, snapshot.meta);
assert_eq!(snapshot_bytes(&mut current).await, bytes);
}
assert_eq!(writer.durable_identity().await.expect("recovered local identity"), identity);
assert_eq!(
logs.read_vote().await.expect("recovered local vote"),
Some(openraft::Vote::new_committed(9, 1))
);
let state = logs.get_log_state().await.expect("coherent recovered log state");
assert!(
state
.last_log_id
.is_some_and(|last| cut.is_some_and(|applied| last.index >= applied.index)),
"SNAPSHOT_INSTALL_LOG_STATE_COVERS_APPLIED_CUT"
);
}
}
#[tokio::test]
async fn native_snapshot_install_ack_survives_power_cut() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let (mut target, backend) = media_target().await;
install(&mut target.sm, &mut snapshot).await;
backend.cut();
let Fixture {
directory,
writer,
logs,
sm,
identity,
} = target;
drop(sm);
drop(logs);
drop(writer);
let writer = NativeLogWriter::new(
Store::open(directory.path().join("db"), identity).expect("acknowledged installation survives power cut"),
);
let mut logs = NativeRaftLogStore::new(writer.clone());
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened durable SM");
let mut current = sm
.get_current_snapshot()
.await
.expect("durable current snapshot")
.expect("present");
assert_eq!(current.meta, snapshot.meta, "SNAPSHOT_INSTALL_ACK_COMPLETE_META_DURABLE");
assert_eq!(snapshot_bytes(&mut current).await, bytes);
assert_eq!(
writer.created_events_page(None, 256).await.expect("durable installed events"),
source.page().await
);
logs.blocking_append(vec![entry(3)]).await.expect("append after ACK recovery");
sm.apply(vec![entry(3)]).await.expect("apply after ACK recovery");
assert_eq!(sm.applied_state().await.expect("continued state").0, Some(id(3).into()));
}
#[tokio::test]
async fn native_snapshot_install_purge_without_pending_rejected() {
let mut target = Fixture::new(1);
target.apply(entries(1)).await;
let before = target.page().await;
let bytes = std::fs::read(target.directory.path().join("db")).expect("old unauthorized purge bytes");
let result = tokio::time::timeout(std::time::Duration::from_secs(2), target.logs.purge(id(3).into()))
.await
.expect("no pending receiver means purge cannot wait for hypothetical installation");
assert!(result.is_err(), "SNAPSHOT_PURGE_WITHOUT_PENDING_FAILS_IMMEDIATELY");
assert_eq!(target.page().await, before);
assert_eq!(
std::fs::read(target.directory.path().join("db")).expect("rejected purge preserves bytes"),
bytes
);
}
async fn wait_for_pending_purge(writer: &NativeLogWriter) {
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while writer.snapshot_receives.waiters.load(std::sync::atomic::Ordering::Acquire) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("actual trait purge reached pending-receive wait");
}
#[tokio::test]
async fn native_snapshot_install_pending_purge_waits_for_install() {
let mut source = Fixture::new(0);
source.apply(entries(3)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let mut target = Fixture::new(1);
target.apply(entries(1)).await;
let data = receive(&mut target.sm, &bytes).await;
let mut purging = target.logs.clone();
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
wait_for_pending_purge(&target.writer).await;
assert!(!purge.is_finished(), "SNAPSHOT_PURGE_WAITS_FOR_INCOMING_CUT_BEYOND_LOCAL_TAIL");
target
.sm
.install_snapshot(&snapshot.meta, data)
.await
.expect("actual install while OpenRaft-style purge waits");
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
.await
.expect("pending purge wakes after install")
.expect("purge task")
.expect("installed checkpoint now authorizes exact purge");
assert_eq!(
target
.logs
.get_log_state()
.await
.expect("installed log state")
.last_purged_log_id,
Some(id(3).into()),
"SNAPSHOT_PENDING_PURGE_USES_DURABLE_INSTALLED_CUT"
);
assert_eq!(target.page().await, source.page().await);
assert_eq!(
target
.writer
.snapshot_receives
.waiters
.load(std::sync::atomic::Ordering::Acquire),
0,
"SNAPSHOT_PURGE_WAIT_OWNER_RELEASED"
);
}
#[tokio::test]
async fn native_snapshot_install_cancelled_receive_wakes_purge() {
let mut target = Fixture::new(1);
target.apply(entries(1)).await;
let page = target.page().await;
let before = std::fs::read(target.directory.path().join("db")).expect("stable target before cancelled receive");
let data = target.sm.begin_receiving_snapshot().await.expect("actual pending receiver");
let mut purging = target.logs.clone();
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
wait_for_pending_purge(&target.writer).await;
assert!(!purge.is_finished(), "SNAPSHOT_CANCELLED_RECEIVE_TEST_HAS_REAL_WAITING_PURGE");
drop(data);
assert!(
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
.await
.expect("cancelled receive wakes pending purge")
.expect("purge task")
.is_err(),
"SNAPSHOT_CANCELLED_RECEIVE_CANNOT_AUTHORIZE_PURGE"
);
assert_eq!(target.page().await, page);
assert_eq!(
std::fs::read(target.directory.path().join("db")).expect("retained cancelled-receive state"),
before
);
assert_eq!(
target
.writer
.snapshot_receives
.waiters
.load(std::sync::atomic::Ordering::Acquire),
0
);
}
#[tokio::test]
async fn native_snapshot_install_failed_receive_wakes_purge() {
let mut source = Fixture::new(0);
source.apply(entries(3)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let mut target = Fixture::new(1);
target.apply(entries(1)).await;
let page = target.page().await;
let before = std::fs::read(target.directory.path().join("db")).expect("stable target before failed install");
let data = receive(&mut target.sm, &bytes).await;
let mut purging = target.logs.clone();
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
wait_for_pending_purge(&target.writer).await;
assert!(!purge.is_finished(), "SNAPSHOT_FAILED_INSTALL_TEST_HAS_REAL_WAITING_PURGE");
let mut wrong = snapshot.meta.clone();
wrong.last_log_id = Some(LogId { term: 9, ..id(3) }.into());
assert!(
target.sm.install_snapshot(&wrong, data).await.is_err(),
"SNAPSHOT_INVALID_META_INSTALL_REJECTED"
);
assert!(
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
.await
.expect("failed install wakes pending purge")
.expect("purge task")
.is_err(),
"SNAPSHOT_FAILED_INSTALL_CANNOT_AUTHORIZE_PURGE"
);
assert_eq!(target.page().await, page);
assert_eq!(
std::fs::read(target.directory.path().join("db")).expect("retained failed-install state"),
before
);
assert_eq!(
target
.writer
.snapshot_receives
.waiters
.load(std::sync::atomic::Ordering::Acquire),
0
);
}
#[tokio::test]
async fn native_snapshot_install_completed_between_purge_query_and_wait() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let data_bytes = snapshot_bytes(&mut snapshot).await;
let mut target = Fixture::new(1);
let data = receive(&mut target.sm, &data_bytes).await;
let entered = std::sync::Arc::new(tokio::sync::Notify::new());
let resume = std::sync::Arc::new(tokio::sync::Notify::new());
*target.writer.snapshot_receives.purge_recheck.lock().expect("test pause") = Some((entered.clone(), resume.clone()));
let mut logs = target.logs.clone();
let purge = tokio::spawn(async move { logs.purge(id(2).into()).await });
tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified())
.await
.expect("purge failed coverage under owner before continuation");
target
.sm
.install_snapshot(&snapshot.meta, data)
.await
.expect("install completes before purge continuation");
resume.notify_one();
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
.await
.expect("completed install wake retained")
.expect("purge task")
.expect("old StaleCheckpoint rechecked after install");
assert_eq!(
target.page().await,
source.page().await,
"SNAPSHOT_COMPLETED_INSTALL_RECHECKS_STALE_PURGE"
);
}
#[tokio::test]
async fn native_snapshot_install_empty_baseline_pending_purge() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let mut target = Fixture::new(1);
target.sm.build_snapshot().await.expect("real local empty baseline");
let data = receive(&mut target.sm, &bytes).await;
let mut logs = target.logs.clone();
let purge = tokio::spawn(async move { logs.purge(id(2).into()).await });
wait_for_pending_purge(&target.writer).await;
target
.sm
.install_snapshot(&snapshot.meta, data)
.await
.expect("install supersedes empty baseline");
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
.await
.expect("empty coverage wake")
.expect("purge task")
.expect("new covered purge");
assert_eq!(
target.page().await,
source.page().await,
"SNAPSHOT_EMPTY_BASELINE_IS_UNCOVERED_NOT_CORRUPT"
);
}
@@ -0,0 +1,175 @@
//! OpenRaft log storage backed by the existing native writer and local durable baseline.
use crate::{CaptureRaftEntry, CaptureTypeConfig, NativeLogReader, NativeLogWriter, NativeWriteError, StoreError};
use openraft::{
LogId, OptionalSend, RaftLogReader, StorageError, StorageIOError, Vote,
storage::{LogFlushed, RaftLogStorage},
};
use redb::{ReadableDatabase, ReadableTable};
/// Clones share one native Store, write ordering and checkpoint authority.
#[derive(Clone)]
pub struct NativeRaftLogStore {
writer: NativeLogWriter,
// Last: owner drain is signaled only after this clone releases its writer.
pub(crate) owner_lease: Option<std::sync::Arc<crate::raft_owner::OwnerLease>>,
}
impl NativeRaftLogStore {
pub fn new(writer: NativeLogWriter) -> Self {
Self {
writer,
owner_lease: None,
}
}
}
fn error(error: NativeWriteError, operation: fn(openraft::AnyError) -> StorageIOError<u64>) -> StorageError<u64> {
StorageError::IO {
source: operation(openraft::AnyError::new(&error)),
}
}
impl RaftLogReader<CaptureTypeConfig> for NativeRaftLogStore {
async fn try_get_log_entries<RB: std::ops::RangeBounds<u64> + Clone + std::fmt::Debug + OptionalSend>(
&mut self,
range: RB,
) -> Result<Vec<CaptureRaftEntry>, StorageError<u64>> {
let mut reader = self.writer.reader();
reader.owner_lease = self.owner_lease.clone();
reader.try_get_log_entries(range).await
}
}
impl RaftLogStorage<CaptureTypeConfig> for NativeRaftLogStore {
type LogReader = NativeLogReader;
async fn get_log_state(&mut self) -> Result<openraft::LogState<CaptureTypeConfig>, StorageError<u64>> {
self.writer
.write(|store| {
if !store.writable {
return Err(StoreError::WriteDisabled);
}
let read = store.database.begin_read()?;
let meta = read.open_table(crate::META)?;
let purged = crate::checkpoint_baseline::read_purged(&meta)?;
let logs = read.open_table(crate::LOGS)?;
let last = logs
.last()?
.map(|(index, value)| {
let id = crate::decode_log(value.value())?.id;
if id.index != index.value() {
return Err(StoreError::CorruptRecord);
}
Ok(id)
})
.transpose()?
.or(purged);
let tail = meta.get("tail")?.map(|value| crate::decode_id(value.value())).transpose()?;
if tail != last {
return Err(StoreError::CorruptRecord);
}
Ok(openraft::LogState {
last_purged_log_id: purged.map(Into::into),
last_log_id: last.map(Into::into),
})
})
.await
.map_err(|e| error(e, StorageIOError::read_logs))
}
async fn get_log_reader(&mut self) -> Self::LogReader {
let mut reader = self.writer.reader();
reader.owner_lease = self.owner_lease.clone();
reader
}
async fn save_vote(&mut self, vote: &Vote<u64>) -> Result<(), StorageError<u64>> {
self.writer
.save_vote(*vote)
.await
.map_err(|e| error(e, StorageIOError::write_vote))
}
async fn read_vote(&mut self) -> Result<Option<Vote<u64>>, StorageError<u64>> {
self.writer.read_vote().await.map_err(|e| error(e, StorageIOError::read_vote))
}
async fn save_committed(&mut self, committed: Option<LogId<u64>>) -> Result<(), StorageError<u64>> {
self.writer
.write(move |store| match committed {
Some(id) => store.save_committed(id.into()),
None if store.read_committed()?.is_none() => Ok(()),
None => Err(StoreError::CommittedRegression),
})
.await
.map_err(|e| error(e, StorageIOError::write_logs))
}
async fn read_committed(&mut self) -> Result<Option<LogId<u64>>, StorageError<u64>> {
self.writer
.write(|store| store.read_committed().map(|id| id.map(Into::into)))
.await
.map_err(|e| error(e, StorageIOError::read_logs))
}
async fn append<I>(&mut self, entries: I, callback: LogFlushed<CaptureTypeConfig>) -> Result<(), StorageError<u64>>
where
I: IntoIterator<Item = CaptureRaftEntry> + OptionalSend,
I::IntoIter: OptionalSend,
{
let entries = entries.into_iter().collect::<Vec<_>>();
// The original blocking worker owns ordering and callback through durable commit,
// even if OpenRaft cancels its awaiting append future.
self.writer
.write(move |store| {
let result = store.append_native(&entries);
callback.log_io_completed(
result
.as_ref()
.map(|_| ())
.map_err(|error| std::io::Error::other(error.code())),
);
result
})
.await
.map_err(|e| error(e, StorageIOError::write_logs))
}
async fn truncate(&mut self, log_id: LogId<u64>) -> Result<(), StorageError<u64>> {
self.writer
.truncate_suffix(log_id.index)
.await
.map_err(|e| error(e, StorageIOError::write_logs))
}
async fn purge(&mut self, log_id: LogId<u64>) -> Result<(), StorageError<u64>> {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30);
loop {
if tokio::time::Instant::now() >= deadline {
return Err(error(NativeWriteError::Store(StoreError::StaleCheckpoint), StorageIOError::write_logs));
}
let notified = self.writer.snapshot_receives.changed.notified();
tokio::pin!(notified);
notified.as_mut().enable();
let receives = self.writer.snapshot_receives.clone();
// Observe coverage and receive ownership while install is excluded by ordering.
let (result, pending) = self
.writer
.write(move |store| Ok((store.purge_native_prefix(log_id.into(), None), receives.pending())))
.await
.map_err(|e| error(e, StorageIOError::write_logs))?;
if matches!(&result, Err(StoreError::StaleCheckpoint)) {
#[cfg(test)]
{
let hook = self
.writer
.snapshot_receives
.purge_recheck
.lock()
.expect("test hook lock")
.take();
if let Some((entered, resume)) = hook {
entered.notify_one();
resume.notified().await;
}
}
if pending {
#[cfg(test)]
let _waiter = crate::raft_state_machine::Waiter::new(self.writer.snapshot_receives.clone());
if tokio::time::timeout_at(deadline, notified).await.is_err() {
return Err(error(NativeWriteError::Store(StoreError::StaleCheckpoint), StorageIOError::write_logs));
}
continue;
}
}
return result.map_err(|e| error(e.into(), StorageIOError::write_logs));
}
}
}
@@ -0,0 +1,439 @@
//! Exercises actual OpenRaft storage traits, not a substitute consensus engine.
use super::*;
use openraft::{
RaftLogReader,
storage::{RaftLogStorage, RaftLogStorageExt},
};
use std::collections::{BTreeMap, BTreeSet};
fn identity() -> NodeIdentity {
NodeIdentity {
cluster_id: [7; 16],
node_id: 0,
}
}
fn binding() -> CaptureBinding {
CaptureBinding {
revision: 9,
binding_id: [8; 16],
target_id: [9; 16],
}
}
fn id(index: u64) -> LogId {
LogId {
term: 8,
leader_node: 0,
index,
}
}
fn entry(index: u64) -> CaptureRaftEntry {
let payload = if index == 0 {
let membership = CaptureMembership::new(
Some(id(0)),
vec![BTreeSet::from([0])],
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("test member"))]),
)
.expect("native membership");
let membership = openraft::StoredMembership::try_from(&membership).expect("stored native membership");
openraft::EntryPayload::Membership(membership.membership().clone())
} else {
let byte = u8::try_from(index).expect("small test event index");
let command = DecideCreated {
operation_id: [byte; 16],
object: ObjectIdentity {
bucket_incarnation: [2; 16],
key: vec![byte],
},
expected_head: None,
prepared: PreparedIdentity {
version_id: [byte; 16],
preparation_id: [4; 16],
content_digest: [5; 32],
content_length: 123,
},
binding_revision: binding().revision,
};
openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("native test command"))
};
CaptureRaftEntry {
log_id: id(index).into(),
payload,
}
}
fn entries(end: u64) -> Vec<CaptureRaftEntry> {
(0..=end).map(entry).collect()
}
fn fixture() -> (tempfile::TempDir, NativeLogWriter, NativeRaftLogStore) {
let directory = tempfile::tempdir().expect("actual native database directory");
let mut store = Store::initialize(directory.path().join("db"), identity()).expect("actual redb initialization");
store.pin_capture_binding(binding()).expect("capture binding");
store.enable_native_empty().expect("native mode");
store.promote_created_provenance().expect("native provenance");
let writer = NativeLogWriter::new(store);
let adapter = NativeRaftLogStore::new(writer.clone());
(directory, writer, adapter)
}
fn media_fixture() -> (tempfile::TempDir, tests::CrashBackend, NativeLogWriter, NativeRaftLogStore) {
let (directory, writer, adapter) = fixture();
drop(adapter);
drop(writer);
let backend = tests::CrashBackend::new(&directory.path().join("db"));
let store = Store::from_database(backend.database(), identity()).expect("existing real disk fault backend");
let writer = NativeLogWriter::new(store);
let adapter = NativeRaftLogStore::new(writer.clone());
(directory, backend, writer, adapter)
}
async fn logs(adapter: &mut NativeRaftLogStore) -> Vec<CaptureRaftEntry> {
adapter
.get_log_reader()
.await
.try_get_log_entries(..)
.await
.expect("actual trait reader")
}
async fn checkpoint(writer: &NativeLogWriter, adapter: &mut NativeRaftLogStore) -> CheckpointReceipt {
let batch = entries(2);
adapter
.blocking_append(batch.clone())
.await
.expect("trait append before checkpoint");
// Prepares genuine applied/checkpoint evidence; this does not claim Raft consensus.
writer
.apply_committed(batch)
.await
.expect("trusted test owner applies preparation entries");
writer
.checkpoint_local(CheckpointLimits::default())
.await
.expect("real durable baseline")
}
#[tokio::test]
async fn raft_log_storage_flush_restart() {
let (directory, backend, writer, mut adapter) = media_fixture();
let batch = entries(2);
// blocking_append constructs the real library-private LogFlushed callback.
adapter
.blocking_append(batch.clone())
.await
.expect("real durable flush callback");
let state = adapter.get_log_state().await.expect("trait log state");
assert_eq!(state.last_log_id, Some(id(2).into()), "RAFT_LOG_TAIL_NOT_APPLIED_STATE");
assert_eq!(state.last_purged_log_id, None);
assert_eq!(writer.applied_state().await.expect("application state").0, None);
assert_eq!(logs(&mut adapter).await, batch);
backend.cut();
drop(adapter);
drop(writer);
let reopened = Store::open(directory.path().join("db"), identity()).expect("reopen after successful callback and power cut");
let mut adapter = NativeRaftLogStore::new(NativeLogWriter::new(reopened));
assert_eq!(logs(&mut adapter).await, batch, "RAFT_LOG_FLUSH_CALLBACK_DURABLE");
assert_eq!(
adapter.get_log_state().await.expect("reopened trait state").last_log_id,
Some(id(2).into())
);
}
#[tokio::test]
async fn raft_log_storage_vote_restart() {
let (directory, backend, writer, mut adapter) = media_fixture();
let vote = openraft::Vote::new_committed(9, 0);
adapter.save_vote(&vote).await.expect("trait durable vote");
assert_eq!(adapter.read_vote().await.expect("trait vote read"), Some(vote));
adapter.blocking_append(entries(2)).await.expect("trait append");
adapter
.save_committed(Some(id(1).into()))
.await
.expect("trait durable commit marker");
assert!(adapter.save_committed(None).await.is_err(), "RAFT_LOG_COMMITTED_NONE_CANNOT_RESET");
assert_eq!(adapter.read_committed().await.expect("exact marker"), Some(id(1).into()));
assert!(
adapter.save_committed(Some(id(0).into())).await.is_err(),
"RAFT_LOG_COMMITTED_NO_REGRESSION"
);
let mut wrong = id(2);
wrong.term += 1;
assert!(
adapter.save_committed(Some(wrong.into())).await.is_err(),
"RAFT_LOG_COMMITTED_EXACT_IDENTITY"
);
backend.cut();
drop(adapter);
drop(writer);
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
Store::open(directory.path().join("db"), identity()).expect("durable vote reopen"),
));
assert_eq!(
reopened.read_vote().await.expect("reopened vote"),
Some(vote),
"RAFT_LOG_VOTE_ACK_DURABLE"
);
assert_eq!(reopened.read_committed().await.expect("reopened committed marker"), Some(id(1).into()));
}
#[tokio::test]
async fn raft_log_storage_partial_purge_restart() {
let (directory, writer, mut adapter) = fixture();
let receipt = checkpoint(&writer, &mut adapter).await;
adapter
.blocking_append(vec![entry(3), entry(4)])
.await
.expect("uncovered suffix append");
let events = writer
.created_events_page(None, 256)
.await
.expect("pre-purge authoritative events");
adapter
.purge(id(1).into())
.await
.expect("checkpoint authorizes strict prefix");
assert_eq!(
logs(&mut adapter).await,
vec![entry(2), entry(3), entry(4)],
"RAFT_LOG_PARTIAL_PURGE_RETAINS_SUFFIX"
);
let state = adapter.get_log_state().await.expect("partially purged state");
assert_eq!(state.last_purged_log_id, Some(id(1).into()));
assert_eq!(state.last_log_id, Some(id(4).into()));
adapter.purge(id(1).into()).await.expect("repeat exact purge is idempotent");
assert!(adapter.purge(id(0).into()).await.is_err(), "RAFT_LOG_PURGE_NO_REGRESSION");
{
let store = writer.store.lock().expect("inspect task-owned store");
let tx = store.database.begin_read().expect("real persisted prefix inspection");
let results = tx.open_table(application::APPLY_RESULTS).expect("apply receipts");
assert!(results.get(0).expect("membership receipt").is_none());
assert!(results.get(1).expect("first receipt").is_none());
assert!(
results.get(2).expect("retained receipt").is_some(),
"RAFT_LOG_PARTIAL_PURGE_RECEIPT_SUFFIX"
);
}
drop(adapter);
drop(writer);
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity()).expect("restart partial purge"));
let mut adapter = NativeRaftLogStore::new(writer.clone());
assert_eq!(
logs(&mut adapter).await,
vec![entry(2), entry(3), entry(4)],
"RAFT_LOG_PARTIAL_PURGE_REOPEN"
);
adapter
.blocking_append(vec![entry(5)])
.await
.expect("continue append after partial purge");
writer
.apply_committed(vec![entry(3), entry(4), entry(5)])
.await
.expect("continue application from true applied state");
assert_eq!(
writer
.created_events_page(None, 256)
.await
.expect("retained plus new events")
.events
.len(),
5,
"RAFT_LOG_PURGE_NEVER_DELETES_EVENTS"
);
writer
.purge_checkpoint(&receipt)
.await
.expect("existing exact receipt API remains compatible after partial purge");
assert_eq!(
adapter
.get_log_state()
.await
.expect("exact compatibility state")
.last_purged_log_id,
Some(id(2).into())
);
assert_eq!(logs(&mut adapter).await, vec![entry(3), entry(4), entry(5)]);
assert_eq!(
writer
.created_events_page(None, 2)
.await
.expect("original events retained")
.events,
events.events
);
drop(adapter);
drop(writer);
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
Store::open(directory.path().join("db"), identity()).expect("reopen continued apply and exact purge"),
));
assert_eq!(logs(&mut reopened).await, vec![entry(3), entry(4), entry(5)]);
}
#[tokio::test]
async fn raft_log_storage_purge_rejects_uncovered() {
let (directory, writer, mut adapter) = fixture();
adapter
.blocking_append(entries(2))
.await
.expect("trait append without snapshot authority");
writer
.apply_committed(entries(2))
.await
.expect("applied preparation without checkpoint");
let before = std::fs::read(directory.path().join("db")).expect("stable pre-rejection bytes");
assert!(
adapter.purge(id(1).into()).await.is_err(),
"PURGE_REQUIRES_CHECKPOINT_COVERAGE: application progress is not a durable checkpoint"
);
assert_eq!(
std::fs::read(directory.path().join("db")).expect("rejected purge bytes"),
before,
"RAFT_LOG_UNAUTHORIZED_PURGE_NO_MUTATION"
);
writer
.checkpoint_local(CheckpointLimits::default())
.await
.expect("checkpoint at applied cut 2");
adapter
.blocking_append(vec![entry(3)])
.await
.expect("log beyond checkpoint cut");
let before = std::fs::read(directory.path().join("db")).expect("post-checkpoint bytes");
assert!(
adapter.purge(id(3).into()).await.is_err(),
"PURGE_REQUIRES_CHECKPOINT_COVERAGE: retained target exceeds durable cut"
);
assert_eq!(
std::fs::read(directory.path().join("db")).expect("uncovered target rejection bytes"),
before,
"RAFT_LOG_UNCOVERED_PURGE_NO_MUTATION"
);
assert_eq!(logs(&mut adapter).await, entries(3));
assert_eq!(adapter.get_log_state().await.expect("unchanged marker").last_purged_log_id, None);
}
#[tokio::test]
async fn raft_log_storage_purge_identity() {
let (directory, writer, mut adapter) = fixture();
checkpoint(&writer, &mut adapter).await;
for target in [
LogId { term: 9, ..id(1) },
LogId { leader_node: 1, ..id(1) },
LogId { term: 9, ..id(2) },
] {
let before = std::fs::read(directory.path().join("db")).expect("before wrong complete LogId");
assert!(adapter.purge(target.into()).await.is_err(), "RAFT_LOG_PURGE_EXACT_LOG_ID");
assert_eq!(std::fs::read(directory.path().join("db")).expect("after identity rejection"), before);
}
adapter.purge(id(1).into()).await.expect("valid identity");
let mut wrong = id(1);
wrong.term += 1;
assert!(adapter.purge(wrong.into()).await.is_err(), "RAFT_LOG_PURGED_REPEAT_IDENTITY_BOUND");
assert_eq!(
adapter
.get_log_state()
.await
.expect("unchanged identity marker")
.last_purged_log_id,
Some(id(1).into())
);
adapter.purge(id(2).into()).await.expect("final exact covered target");
let state = adapter.get_log_state().await.expect("no remaining logs");
assert_eq!(state.last_purged_log_id, Some(id(2).into()));
assert_eq!(state.last_log_id, Some(id(2).into()), "RAFT_LOG_EMPTY_TAIL_USES_PURGED_LOG_ID");
assert!(logs(&mut adapter).await.is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn raft_log_storage_cancelled_append() {
let (directory, backend, writer, adapter) = media_fixture();
let (entered, resume) = backend.pause_sync(1, false);
let mut first = adapter.clone();
let task = tokio::spawn(async move { first.blocking_append(entries(2)).await });
entered.wait(); // Actual redb commit has begun but its first sync is paused.
assert!(!task.is_finished(), "RAFT_LOG_FLUSH_NOT_BEFORE_DURABILITY");
task.abort();
assert!(task.await.expect_err("cancelled blocking append waiter").is_cancelled());
assert!(writer.ordering.try_lock().is_err(), "RAFT_LOG_CANCELLED_APPEND_RETAINS_ORDERING_OWNER");
let mut following = adapter.clone();
let next = tokio::spawn(async move { following.blocking_append(vec![entry(3)]).await });
assert!(writer.ordering.try_lock().is_err(), "RAFT_LOG_NEXT_APPEND_CANNOT_BYPASS_PENDING_SYNC");
resume.wait();
next.await.expect("next trait append task").expect("ordered follow-up append");
let mut adapter = adapter;
assert_eq!(
logs(&mut adapter).await,
entries(3),
"RAFT_LOG_CANCELLED_WAITER_BACKGROUND_COMMIT_COMPLETE"
);
backend.cut();
drop(adapter);
drop(writer);
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
Store::open(directory.path().join("db"), identity()).expect("reopen cancelled waiter background commit"),
));
assert_eq!(logs(&mut reopened).await, entries(3));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn raft_log_storage_failed_append() {
let (directory, backend, writer, adapter) = media_fixture();
let (entered, resume) = backend.pause_sync(1, false);
let mut appending = adapter.clone();
let task = tokio::spawn(async move { appending.blocking_append(entries(2)).await });
entered.wait();
assert!(!task.is_finished(), "RAFT_LOG_FAILED_FLUSH_NOT_EARLY_SUCCESS");
backend.cut();
resume.wait();
assert!(
task.await.expect("failed actual trait append task").is_err(),
"RAFT_LOG_FAILED_APPEND_NEVER_REPORTS_CALLBACK_SUCCESS"
);
let mut adapter = adapter;
assert!(
adapter.blocking_append(entries(2)).await.is_err(),
"RAFT_LOG_INDETERMINATE_STORE_REJECTS_LATER_APPEND"
);
assert!(
adapter.save_vote(&openraft::Vote::new_committed(9, 0)).await.is_err(),
"RAFT_LOG_INDETERMINATE_STORE_REJECTS_LATER_VOTE"
);
drop(adapter);
drop(writer);
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
Store::open(directory.path().join("db"), identity()).expect("reopen failed-before-first-sync append"),
));
assert!(logs(&mut reopened).await.is_empty(), "RAFT_LOG_FAILURE_NO_PARTIAL_BATCH");
}
#[tokio::test]
async fn raft_log_storage_truncate_uncommitted_suffix() {
let (directory, writer, mut adapter) = fixture();
adapter.blocking_append(entries(4)).await.expect("trait append full log");
adapter
.save_committed(Some(id(2).into()))
.await
.expect("protect committed prefix");
assert!(
adapter.truncate(id(2).into()).await.is_err(),
"RAFT_LOG_TRUNCATE_PROTECTS_COMMITTED_PREFIX"
);
adapter
.truncate(id(3).into())
.await
.expect("truncate includes uncommitted target");
assert_eq!(logs(&mut adapter).await, entries(2), "RAFT_LOG_TRUNCATE_INCLUSIVE");
let mut replacement = entry(3);
replacement.log_id = LogId { term: 9, ..id(3) }.into();
adapter
.blocking_append(vec![replacement.clone()])
.await
.expect("trait append replacement suffix");
assert_eq!(
adapter.get_log_state().await.expect("replacement tail").last_log_id,
Some(replacement.log_id)
);
drop(adapter);
drop(writer);
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
Store::open(directory.path().join("db"), identity()).expect("reopen truncate and replacement"),
));
let mut expected = entries(2);
expected.push(replacement);
assert_eq!(logs(&mut reopened).await, expected);
}
+251
View File
@@ -0,0 +1,251 @@
//! Explicit lifecycle for real OpenRaft over one durable native capture owner.
use crate::{
CaptureCommand, CaptureNode, CaptureNodeError, CaptureTypeConfig, CheckpointLimits, NativeLogWriter, NativeRaftLogStore,
NativeRaftStateMachine, NativeSnapshotError, NativeWriteError, StoreError,
};
use openraft::{
Config, ConfigError, Raft, RaftMetrics, RaftNetworkFactory,
error::{ClientWriteError, Fatal, InitializeError, RaftError},
raft::ClientWriteResponse,
};
use redb::ReadableDatabase;
use std::{
collections::BTreeMap,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
use tokio::sync::watch;
#[derive(Debug, thiserror::Error)]
pub enum NativeRaftOwnerError {
#[error(transparent)]
Config(#[from] Box<ConfigError>),
#[error(transparent)]
Store(#[from] NativeWriteError),
#[error(transparent)]
Snapshot(#[from] NativeSnapshotError),
#[error("runtime node ID {runtime} differs from durable node ID {durable}")]
IdentityMismatch { runtime: u64, durable: u64 },
#[error("bootstrap must contain the local node")]
MissingLocalMember,
#[error(transparent)]
Node(#[from] CaptureNodeError),
#[error("native Raft owner is stopping or stopped")]
Stopped,
#[error(transparent)]
Fatal(Box<Fatal<u64>>),
#[error(transparent)]
Initialize(Box<RaftError<u64, InitializeError<u64, CaptureNode>>>),
#[error(transparent)]
Write(Box<RaftError<u64, ClientWriteError<u64, CaptureNode>>>),
#[error("native Raft shutdown task failed")]
Shutdown(#[source] tokio::task::JoinError),
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[error(transparent)]
Reader(#[from] crate::source_reader::ReaderError),
}
#[derive(Default)]
struct DrainState {
complete: AtomicBool,
changed: tokio::sync::Notify,
#[cfg(test)]
waiters: Arc<std::sync::atomic::AtomicUsize>,
}
impl DrainState {
async fn wait(&self) {
#[cfg(test)]
let _waiter = DrainWaiter::new(self.waiters.clone());
loop {
let notified = self.changed.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self.complete.load(Ordering::Acquire) {
return;
}
notified.await;
}
}
}
#[cfg(test)]
struct DrainWaiter(Arc<std::sync::atomic::AtomicUsize>);
#[cfg(test)]
impl DrainWaiter {
fn new(waiters: Arc<std::sync::atomic::AtomicUsize>) -> Self {
waiters.fetch_add(1, Ordering::AcqRel);
Self(waiters)
}
}
#[cfg(test)]
impl Drop for DrainWaiter {
fn drop(&mut self) {
self.0.fetch_sub(1, Ordering::AcqRel);
}
}
// The real worker, snapshot builders and replication readers share this one lease.
// Its final drop runs after their NativeLogWriter fields have been released.
pub(crate) struct OwnerLease {
drain: Arc<DrainState>,
}
impl Drop for OwnerLease {
fn drop(&mut self) {
self.drain.complete.store(true, Ordering::Release);
self.drain.changed.notify_waiters();
}
}
/// A lifecycle owner, not a transport. All storage adapters share its original writer.
pub struct NativeRaftOwner {
raft: Raft<CaptureTypeConfig>,
writer: NativeLogWriter,
identity: crate::NodeIdentity,
accepting: AtomicBool,
drain: Arc<DrainState>,
#[cfg(test)]
pub(crate) snapshot_build_pause: Arc<crate::raft_state_machine::TestPause>,
}
impl NativeRaftOwner {
pub async fn start<N>(
writer: NativeLogWriter,
node_id: u64,
config: Arc<Config>,
limits: CheckpointLimits,
network: N,
) -> Result<Self, NativeRaftOwnerError>
where
N: RaftNetworkFactory<CaptureTypeConfig>,
{
// All operator-input checks precede Raft::new and its background workers.
let config = Arc::new(
(*config)
.clone()
.validate()
.map_err(|error| NativeRaftOwnerError::Config(Box::new(error)))?,
);
let mut state_machine = NativeRaftStateMachine::new(writer.clone(), limits)?;
let identity = writer
.write(|store| {
if !store.writable {
return Err(StoreError::WriteDisabled);
}
let transaction = store.database.begin_read()?;
let meta = transaction.open_table(crate::META)?;
crate::application::native_application_state(&transaction)?;
crate::application::read_binding(&meta)?;
crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())
})
.await?;
if identity.node_id != node_id {
return Err(NativeRaftOwnerError::IdentityMismatch {
runtime: node_id,
durable: identity.node_id,
});
}
let drain = Arc::new(DrainState::default());
let lease = Arc::new(OwnerLease { drain: drain.clone() });
state_machine.owner_lease = Some(lease.clone());
let mut logs = NativeRaftLogStore::new(writer.clone());
logs.owner_lease = Some(lease);
#[cfg(test)]
let snapshot_build_pause = state_machine.build_before_ordering.clone();
let raft = Raft::new(node_id, config, network, logs, state_machine)
.await
.map_err(|error| NativeRaftOwnerError::Fatal(Box::new(error)))?;
Ok(Self {
raft,
writer,
identity,
accepting: AtomicBool::new(true),
drain,
#[cfg(test)]
snapshot_build_pause,
})
}
/// Identity read from durable META and checked before starting this owner.
pub fn identity(&self) -> crate::NodeIdentity {
self.identity
}
fn require_running(&self) -> Result<(), NativeRaftOwnerError> {
if self.accepting.load(Ordering::Acquire) {
Ok(())
} else {
Err(NativeRaftOwnerError::Stopped)
}
}
/// Only explicit first-cluster formation calls initialize; start never does.
pub async fn bootstrap(&self, members: BTreeMap<u64, CaptureNode>) -> Result<(), NativeRaftOwnerError> {
self.require_running()?;
if !members.contains_key(&self.identity.node_id) {
return Err(NativeRaftOwnerError::MissingLocalMember);
}
for node in members.values() {
node.validate()?;
}
self.raft
.initialize(members)
.await
.map_err(|error| NativeRaftOwnerError::Initialize(Box::new(error)))
}
/// The returned LogId and application response come from actual consensus apply.
pub async fn client_write(
&self,
command: CaptureCommand,
) -> Result<ClientWriteResponse<CaptureTypeConfig>, NativeRaftOwnerError> {
self.require_running()?;
self.raft
.client_write(command)
.await
.map_err(|error| NativeRaftOwnerError::Write(Box::new(error)))
}
/// Re-read an exact successful first decision from this owner's durable view.
/// Use the Created result's decision LogId, not an idempotent retry entry's outer LogId.
pub async fn authorize_created(
&self,
expected: crate::DecideCreated,
expected_decision: crate::LogId,
) -> Result<crate::CommittedCreatedAuthorization, NativeRaftOwnerError> {
self.require_running()?;
Ok(self.writer.authorize_created(expected, expected_decision).await?)
}
pub fn metrics(&self) -> watch::Receiver<RaftMetrics<u64, CaptureNode>> {
self.raft.metrics()
}
/// Transport registration only; raw clones remain trusted caller-owned handles.
pub fn raft(&self) -> &Raft<CaptureTypeConfig> {
&self.raft
}
/// The caller owns serve/shutdown and must drain it before releasing the Store.
#[cfg(any(target_os = "linux", target_os = "macos"))]
pub fn bind_reader(
&self,
path: impl AsRef<std::path::Path>,
) -> Result<(crate::source_reader::SourceReaderServer, crate::source_reader::Shutdown), NativeRaftOwnerError> {
self.require_running()?;
Ok(crate::source_reader::SourceReaderServer::bind(path, self.writer.clone())?)
}
#[cfg(test)]
pub(crate) fn shutdown_waiters(&self) -> Arc<std::sync::atomic::AtomicUsize> {
self.drain.waiters.clone()
}
/// Stop admission before signaling Raft; await its real core/tick shutdown.
pub async fn shutdown(&self) -> Result<(), NativeRaftOwnerError> {
self.accepting.store(false, Ordering::Release);
self.raft.shutdown().await.map_err(NativeRaftOwnerError::Shutdown)?;
// OpenRaft joins core/tick but detaches the SM worker and snapshot builders.
self.drain.wait().await;
Ok(())
}
}
@@ -0,0 +1,885 @@
//! Real Raft engines and real redb; direct RPC routing exists only inside these tests.
use super::*;
use openraft::{
Raft, RaftNetwork, RaftNetworkFactory, ServerState,
error::{InstallSnapshotError, RPCError, RaftError, RemoteError, Unreachable},
network::{Backoff, RPCOption},
raft::{
AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, VoteRequest, VoteResponse,
},
};
use std::{
collections::{BTreeMap, BTreeSet},
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
type Engine = Raft<CaptureTypeConfig>;
type RpcError = RPCError<u64, CaptureNode, RaftError<u64>>;
type SnapshotRpcError = RPCError<u64, CaptureNode, RaftError<u64, InstallSnapshotError>>;
type PausedLinks = Mutex<BTreeMap<(u64, u64), (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>;
#[derive(Default)]
struct Routes {
nodes: Mutex<BTreeMap<u64, Engine>>,
blocked: Mutex<BTreeSet<(u64, u64)>>,
snapshot_installs: AtomicUsize,
pause_append: PausedLinks,
}
impl Routes {
fn register(&self, id: u64, raft: Engine) {
self.nodes.lock().expect("routes").insert(id, raft);
}
fn remove(&self, id: u64) {
self.nodes.lock().expect("routes").remove(&id);
}
fn isolate(&self, id: u64, members: &[u64]) {
let mut blocked = self.blocked.lock().expect("partition routes");
for peer in members {
if *peer != id {
blocked.insert((id, *peer));
blocked.insert((*peer, id));
}
}
}
fn heal(&self) {
self.blocked.lock().expect("restore routes").clear();
}
fn endpoint(&self, source: u64, target: u64) -> Result<Engine, Unreachable> {
if self.blocked.lock().expect("partition check").contains(&(source, target)) {
return Err(Unreachable::new(&std::io::Error::other("test partition")));
}
self.nodes
.lock()
.expect("endpoint registry")
.get(&target)
.cloned()
.ok_or_else(|| Unreachable::new(&std::io::Error::other("test endpoint unavailable")))
}
}
#[derive(Clone)]
struct DirectFactory {
source: u64,
routes: Arc<Routes>,
}
struct DirectClient {
source: u64,
target: u64,
node: CaptureNode,
routes: Arc<Routes>,
}
impl RaftNetworkFactory<CaptureTypeConfig> for DirectFactory {
type Network = DirectClient;
async fn new_client(&mut self, target: u64, node: &CaptureNode) -> DirectClient {
DirectClient {
source: self.source,
target,
node: node.clone(),
routes: self.routes.clone(),
}
}
}
impl RaftNetwork<CaptureTypeConfig> for DirectClient {
async fn append_entries(
&mut self,
request: AppendEntriesRequest<CaptureTypeConfig>,
_option: RPCOption,
) -> Result<AppendEntriesResponse<u64>, RpcError> {
let pause = self
.routes
.pause_append
.lock()
.expect("actual RPC pause hook")
.remove(&(self.source, self.target));
if let Some((entered, resume)) = pause {
entered.notify_one();
resume.notified().await;
}
let raft = self.routes.endpoint(self.source, self.target)?;
raft.append_entries(request)
.await
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into())
}
async fn vote(&mut self, request: VoteRequest<u64>, _option: RPCOption) -> Result<VoteResponse<u64>, RpcError> {
let raft = self.routes.endpoint(self.source, self.target)?;
raft.vote(request)
.await
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into())
}
async fn install_snapshot(
&mut self,
request: InstallSnapshotRequest<CaptureTypeConfig>,
_option: RPCOption,
) -> Result<InstallSnapshotResponse<u64>, SnapshotRpcError> {
let raft = self.routes.endpoint(self.source, self.target)?;
let done = request.done;
let result = raft
.install_snapshot(request)
.await
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into());
if done && result.is_ok() {
self.routes.snapshot_installs.fetch_add(1, Ordering::AcqRel);
}
result
}
fn backoff(&self) -> Backoff {
Backoff::new(std::iter::repeat(Duration::from_millis(25)))
}
}
fn identity(node_id: u64) -> NodeIdentity {
NodeIdentity {
cluster_id: [7; 16],
node_id,
}
}
fn binding() -> CaptureBinding {
CaptureBinding {
revision: 9,
binding_id: [8; 16],
target_id: [9; 16],
}
}
fn command(value: u8) -> CaptureCommand {
CaptureCommand::try_created(DecideCreated {
operation_id: [value; 16],
object: ObjectIdentity {
bucket_incarnation: [2; 16],
key: vec![value],
},
expected_head: None,
prepared: PreparedIdentity {
version_id: [value; 16],
preparation_id: [4; 16],
content_digest: [5; 32],
content_length: 123,
},
binding_revision: 9,
})
.expect("native client command")
}
fn config(snapshot_every: u64) -> Arc<openraft::Config> {
Arc::new(
openraft::Config {
cluster_name: "native-owner-tests".into(),
heartbeat_interval: 100,
election_timeout_min: 500,
election_timeout_max: 1000,
snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast(snapshot_every),
max_in_snapshot_log_to_keep: 0,
purge_batch_size: 1,
..openraft::Config::default()
}
.validate()
.expect("valid real Raft test configuration"),
)
}
fn members(ids: &[u64]) -> BTreeMap<u64, CaptureNode> {
ids.iter()
.map(|id| {
(
*id,
CaptureNode::new(format!("http://127.0.0.1:{}", 7000 + id)).expect("bounded test endpoint"),
)
})
.collect()
}
struct Node {
directory: tempfile::TempDir,
writer: NativeLogWriter,
owner: Option<NativeRaftOwner>,
id: u64,
}
impl Node {
async fn start(id: u64, routes: Arc<Routes>, config: Arc<openraft::Config>) -> Self {
let directory = tempfile::tempdir().expect("real Raft owner directory");
let mut store = Store::initialize(directory.path().join("db"), identity(id)).expect("real owner redb");
store.pin_capture_binding(binding()).expect("captured binding");
store.enable_native_empty().expect("native storage mode");
store.promote_created_provenance().expect("native provenance");
let writer = NativeLogWriter::new(store);
let owner = NativeRaftOwner::start(
writer.clone(),
id,
config,
CheckpointLimits::default(),
DirectFactory {
source: id,
routes: routes.clone(),
},
)
.await
.expect("actual owner start");
routes.register(id, owner.raft().clone());
Self {
directory,
writer,
owner: Some(owner),
id,
}
}
fn owner(&self) -> &NativeRaftOwner {
self.owner.as_ref().expect("live owner")
}
async fn page(&self) -> EventPage {
self.writer
.created_events_page(None, 256)
.await
.expect("actual committed events")
}
async fn stop(&mut self, routes: &Routes) {
self.owner
.take()
.expect("running owner")
.shutdown()
.await
.expect("real Raft shutdown");
routes.remove(self.id);
}
}
struct Cluster {
nodes: Vec<Node>,
routes: Arc<Routes>,
}
impl Cluster {
async fn start(ids: &[u64], snapshot_every: u64) -> Self {
let routes = Arc::new(Routes::default());
let mut nodes = Vec::new();
for id in ids {
nodes.push(Node::start(*id, routes.clone(), config(snapshot_every)).await);
}
nodes[0]
.owner()
.bootstrap(members(ids))
.await
.expect("explicit initial cluster bootstrap");
let cluster = Self { nodes, routes };
cluster.leader(ids).await;
cluster
}
fn node(&self, id: u64) -> &Node {
self.nodes.iter().find(|node| node.id == id).expect("cluster node")
}
async fn leader(&self, eligible: &[u64]) -> u64 {
tokio::time::timeout(Duration::from_secs(15), async {
loop {
for id in eligible {
let node = self.node(*id);
let Some(owner) = &node.owner else {
continue;
};
let metrics = owner.metrics().borrow().clone();
assert!(metrics.running_state.is_ok(), "real Raft owner fatal state: {:?}", metrics.running_state);
if metrics.state == ServerState::Leader && metrics.current_leader == Some(*id) {
return *id;
}
}
tokio::task::yield_now().await;
}
})
.await
.expect("real leader election readiness")
}
async fn applied(&self, ids: &[u64], index: u64) {
for id in ids {
self.node(*id)
.owner()
.raft()
.wait(Some(Duration::from_secs(15)))
.applied_index_at_least(Some(index), "real replicated apply readiness")
.await
.expect("committed log applied on real node");
}
}
async fn stop(mut self) {
for node in &mut self.nodes {
if node.owner.is_some() {
node.stop(&self.routes).await;
}
}
}
}
#[tokio::test]
async fn raft_owner_single_node_bootstrap() {
let cluster = Cluster::start(&[0], 100).await;
let response = cluster
.node(0)
.owner()
.client_write(command(1))
.await
.expect("real client_write");
assert!(
matches!(response.data.created_result(),Some(CreatedResult::Created { event_id,.. }) if *event_id==[1;16]),
"OWNER_TRUE_RAFT_CREATED_RESPONSE"
);
let page = cluster.node(0).page().await;
assert_eq!(page.events.len(), 1);
assert_eq!(
page.events[0].captured.decision_log_id,
LogId::from(response.log_id),
"OWNER_COMMITTED_EVENT_LOG_ID"
);
assert!(
cluster.node(0).owner().bootstrap(members(&[0])).await.is_err(),
"OWNER_BOOTSTRAP_NOT_REINITIALIZATION"
);
cluster.stop().await;
}
#[tokio::test]
async fn raft_owner_single_node_restart() {
let routes = Arc::new(Routes::default());
let mut node = Node::start(0, routes.clone(), config(100)).await;
node.owner().bootstrap(members(&[0])).await.expect("first bootstrap only");
node.owner()
.raft()
.wait(Some(Duration::from_secs(15)))
.state(ServerState::Leader, "single leader")
.await
.expect("leader ready");
node.owner().client_write(command(1)).await.expect("first genuine write");
let before = node.page().await;
node.stop(&routes).await;
let Node {
directory,
writer,
owner: _,
id,
} = node;
let vote = writer.read_vote().await.expect("durable vote before reopen");
drop(writer);
let store = Store::open(directory.path().join("db"), identity(id)).expect("restart same durable identity");
assert_eq!(store.read_vote().expect("persisted vote"), vote.map(Into::into));
let writer = NativeLogWriter::new(store);
let owner = NativeRaftOwner::start(
writer.clone(),
id,
config(100),
CheckpointLimits::default(),
DirectFactory {
source: id,
routes: routes.clone(),
},
)
.await
.expect("restart existing Raft");
routes.register(id, owner.raft().clone());
assert!(
owner.raft().is_initialized().await.expect("restored cluster membership"),
"OWNER_RESTART_DOES_NOT_REQUIRE_BOOTSTRAP"
);
owner
.raft()
.wait(Some(Duration::from_secs(15)))
.state(ServerState::Leader, "restored leader")
.await
.expect("restarted leader ready");
owner
.client_write(command(2))
.await
.expect("continue after restart without initialize");
let after = writer.created_events_page(None, 256).await.expect("restart continued events");
assert_eq!(after.events.len(), 2);
assert_eq!(after.events[0], before.events[0], "OWNER_RESTART_PRESERVES_FIRST_DECISION");
owner.shutdown().await.expect("restart owner shutdown");
routes.remove(id);
}
#[tokio::test]
async fn raft_owner_idempotent_retry() {
let cluster = Cluster::start(&[0], 100).await;
let owner = cluster.node(0).owner();
let first = owner.client_write(command(1)).await.expect("first write");
let replay = owner.client_write(command(1)).await.expect("same operation retry");
assert_eq!(
replay.data.created_result(),
first.data.created_result(),
"OWNER_RETRY_PRESERVES_FIRST_DECISION_RESULT"
);
assert!(replay.log_id.index > first.log_id.index, "real retry can append another Raft log");
assert_eq!(cluster.node(0).page().await.events.len(), 1, "OWNER_RETRY_NO_DUPLICATE_EVENT");
cluster.stop().await;
}
#[tokio::test]
async fn raft_owner_identity_binding() {
let routes = Arc::new(Routes::default());
let mut node = Node::start(0, routes.clone(), config(100)).await;
node.owner().bootstrap(members(&[0])).await.expect("real durable membership");
node.owner()
.raft()
.wait(Some(Duration::from_secs(15)))
.state(ServerState::Leader, "single leader")
.await
.expect("leader");
node.owner().client_write(command(1)).await.expect("nonempty durable store");
node.stop(&routes).await;
let before = std::fs::read(node.directory.path().join("db")).expect("stable identity-bound store");
let result = NativeRaftOwner::start(
node.writer.clone(),
1,
config(100),
CheckpointLimits::default(),
DirectFactory {
source: 1,
routes: routes.clone(),
},
)
.await;
assert!(
matches!(result, Err(NativeRaftOwnerError::IdentityMismatch { runtime: 1, durable: 0 })),
"OWNER_REQUIRES_DURABLE_NODE_ID: runtime node cannot reinterpret stored membership/identity"
);
assert_eq!(
std::fs::read(node.directory.path().join("db")).expect("unchanged identity-bound store"),
before
);
let invalid_config = Arc::new(openraft::Config {
election_timeout_min: 1000,
election_timeout_max: 500,
..openraft::Config::default()
});
let invalid = NativeRaftOwner::start(
node.writer.clone(),
0,
invalid_config,
CheckpointLimits::default(),
DirectFactory {
source: 0,
routes: routes.clone(),
},
)
.await;
assert!(
matches!(invalid, Err(NativeRaftOwnerError::Config(_))),
"OWNER_REJECTS_INVALID_CONFIG_BEFORE_START"
);
let mut limits = CheckpointLimits::default();
limits.snapshot.total_bytes = 0;
let invalid = NativeRaftOwner::start(node.writer.clone(), 0, config(100), limits, DirectFactory { source: 0, routes }).await;
assert!(
matches!(invalid, Err(NativeRaftOwnerError::Snapshot(_))),
"OWNER_REJECTS_INVALID_LIMITS_BEFORE_START"
);
assert_eq!(
std::fs::read(node.directory.path().join("db")).expect("invalid start leaves stable bytes"),
before
);
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[tokio::test]
async fn raft_owner_shared_reader() {
use source_reader::SourceReaderClient;
use std::os::unix::fs::PermissionsExt;
let cluster = Cluster::start(&[0], 100).await;
let node = cluster.node(0);
std::fs::set_permissions(node.directory.path(), std::fs::Permissions::from_mode(0o700)).expect("private reader parent");
let socket = node.directory.path().join("reader.sock");
let (server, shutdown) = node
.owner()
.bind_reader(&socket)
.expect("owner shares live writer with reader");
let task = tokio::spawn(server.serve());
let client = SourceReaderClient::new(&socket).expect("live reader");
assert!(
client
.page(identity(0).cluster_id, None, 1)
.await
.expect("initial reader page")
.page
.events
.is_empty()
);
let response = node
.owner()
.client_write(command(1))
.await
.expect("real write while reader runs");
let page = client
.page(identity(0).cluster_id, None, 1)
.await
.expect("live committed event");
assert_eq!(page.identity, identity(0));
assert_eq!(
page.page.events[0].captured.decision_log_id,
LogId::from(response.log_id),
"OWNER_READER_USES_REAL_COMMITTED_WRITER"
);
shutdown.shutdown();
task.await.expect("reader serve task").expect("reader drains");
cluster.stop().await;
}
#[tokio::test]
async fn raft_owner_shutdown() {
let cluster = Cluster::start(&[0], 100).await;
let owner = cluster.node(0).owner();
owner.client_write(command(1)).await.expect("accepted before shutdown");
owner.shutdown().await.expect("actual owner shutdown");
assert!(owner.client_write(command(2)).await.is_err(), "OWNER_SHUTDOWN_REJECTS_NEW_WRITES");
assert_eq!(cluster.node(0).page().await.events.len(), 1);
cluster.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn raft_owner_three_node_majority() {
let cluster = Cluster::start(&[0, 1, 2], 100).await;
let leader = cluster.leader(&[0, 1, 2]).await;
let first = cluster
.node(leader)
.owner()
.client_write(command(1))
.await
.expect("real quorum write");
cluster.applied(&[0, 1, 2], first.log_id.index).await;
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("nonleader");
let rejected = cluster.node(follower).owner().client_write(command(254)).await;
match rejected {
Err(NativeRaftOwnerError::Write(error)) => {
assert!(
matches!(error.api_error(), Some(openraft::error::ClientWriteError::ForwardToLeader(_))),
"OWNER_PRESERVES_TYPED_FORWARD_TO_LEADER"
);
}
other => panic!("OWNER_NONLEADER_WRITE_MUST_NOT_SUCCEED: {other:?}"),
}
cluster.routes.isolate(leader, &[0, 1, 2]);
let uncertain =
tokio::time::timeout(Duration::from_millis(1500), cluster.node(leader).owner().client_write(command(2))).await;
assert!(!matches!(uncertain, Ok(Ok(_))), "OWNER_MAJORITY_REQUIRED_FOR_SUCCESS");
let others: Vec<_> = [0, 1, 2].into_iter().filter(|id| *id != leader).collect();
let next = cluster.leader(&others).await;
let response = cluster
.node(next)
.owner()
.client_write(command(2))
.await
.expect("majority-owned retry of exact same operation");
cluster.routes.heal();
cluster.applied(&[0, 1, 2], response.log_id.index).await;
for node in &cluster.nodes {
assert_eq!(node.page().await.events.len(), 2, "OWNER_QUORUM_RETRY_NO_DUPLICATE_EVENT");
}
cluster.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn raft_owner_leader_failover() {
let mut cluster = Cluster::start(&[0, 1, 2], 100).await;
let leader = cluster.leader(&[0, 1, 2]).await;
let first = cluster
.node(leader)
.owner()
.client_write(command(1))
.await
.expect("first confirmed write");
cluster.applied(&[0, 1, 2], first.log_id.index).await;
let before = cluster.node(leader).page().await.events[0].clone();
cluster
.nodes
.iter_mut()
.find(|node| node.id == leader)
.expect("old leader")
.stop(&cluster.routes)
.await;
let survivors: Vec<_> = [0, 1, 2].into_iter().filter(|id| *id != leader).collect();
let next = cluster.leader(&survivors).await;
let response = cluster
.node(next)
.owner()
.client_write(command(2))
.await
.expect("new leader actual quorum write");
cluster.applied(&survivors, response.log_id.index).await;
for id in survivors {
let page = cluster.node(id).page().await;
assert_eq!(page.events.len(), 2);
assert_eq!(page.events[0], before, "OWNER_FAILOVER_PRESERVES_CONFIRMED_DECISION");
}
cluster.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn raft_owner_snapshot_catchup() {
let cluster = Cluster::start(&[0, 1, 2], 1000).await;
let leader = cluster.leader(&[0, 1, 2]).await;
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("lagging follower");
let first = cluster
.node(leader)
.owner()
.client_write(command(1))
.await
.expect("initial confirmed event");
cluster.applied(&[0, 1, 2], first.log_id.index).await;
// Keep the catch-up test focused on replication rather than an isolated follower's election.
cluster.node(follower).owner().raft().runtime_config().elect(false);
cluster.routes.isolate(follower, &[0, 1, 2]);
let mut last = first;
for value in 2..=12 {
last = cluster
.node(leader)
.owner()
.client_write(command(value))
.await
.expect("majority writes while follower offline");
}
cluster
.node(leader)
.owner()
.raft()
.trigger()
.snapshot()
.await
.expect("true Raft snapshot trigger");
cluster
.node(leader)
.owner()
.raft()
.wait(Some(Duration::from_secs(15)))
.metrics(
|metrics| metrics.purged.is_some_and(|cut| cut.index >= last.log_id.index),
"snapshot durably purged lagging log prefix",
)
.await
.expect("real snapshot and purge");
let before = cluster.routes.snapshot_installs.load(Ordering::Acquire);
cluster.routes.heal();
cluster.applied(&[0, 1, 2], last.log_id.index).await;
cluster
.node(follower)
.owner()
.raft()
.wait(Some(Duration::from_secs(15)))
.metrics(
|metrics| metrics.snapshot.is_some_and(|cut| cut.index >= last.log_id.index),
"follower installed actual transferred snapshot",
)
.await
.expect("real snapshot catch-up");
tokio::time::timeout(Duration::from_secs(15), async {
while cluster.routes.snapshot_installs.load(Ordering::Acquire) <= before {
tokio::task::yield_now().await;
}
})
.await
.expect("snapshot readiness includes actual successful done-RPC return");
assert!(
cluster.routes.snapshot_installs.load(Ordering::Acquire) > before,
"OWNER_CATCHUP_USED_ACTUAL_SNAPSHOT_RPC"
);
let expected = cluster.node(leader).page().await;
for node in &cluster.nodes {
assert_eq!(node.page().await, expected, "OWNER_SNAPSHOT_CATCHUP_FULL_SOURCE_EVENTS");
}
cluster.node(follower).owner().raft().runtime_config().elect(true);
let current = cluster.leader(&[0, 1, 2]).await;
let next = cluster
.node(current)
.owner()
.client_write(command(13))
.await
.expect("continue after real snapshot install");
cluster.applied(&[0, 1, 2], next.log_id.index).await;
for node in &cluster.nodes {
assert_eq!(node.page().await.events.len(), 13);
}
cluster.stop().await;
}
async fn wait_for_shutdown_drain(waiters: &AtomicUsize) {
tokio::time::timeout(Duration::from_secs(5), async {
while waiters.load(Ordering::Acquire) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("OWNER_SHUTDOWN_DRAINS_INFLIGHT_WORK: owner reached real worker drain wait");
}
#[tokio::test]
async fn raft_owner_shutdown_inflight_snapshot() {
let routes = Arc::new(Routes::default());
let mut node = Node::start(0, routes.clone(), config(1000)).await;
node.owner()
.bootstrap(members(&[0]))
.await
.expect("real single-node bootstrap");
node.owner()
.raft()
.wait(Some(Duration::from_secs(15)))
.state(ServerState::Leader, "leader before snapshot")
.await
.expect("leader readiness");
node.owner()
.client_write(command(1))
.await
.expect("real committed snapshot input");
let entered = Arc::new(tokio::sync::Notify::new());
let resume = Arc::new(tokio::sync::Notify::new());
*node.owner().snapshot_build_pause.lock().expect("real builder hook") = Some((entered.clone(), resume.clone()));
node.owner()
.raft()
.trigger()
.snapshot()
.await
.expect("real Raft snapshot request");
tokio::time::timeout(Duration::from_secs(5), entered.notified())
.await
.expect("real detached snapshot builder paused");
let owner = node.owner.take().expect("running owner");
let waiters = owner.shutdown_waiters();
let observer = owner.raft().clone();
let shutdown = tokio::spawn(async move {
let result = owner.shutdown().await;
(owner, result)
});
wait_for_shutdown_drain(&waiters).await;
assert!(
observer.metrics().borrow().running_state.is_err(),
"OWNER_CORE_STOPPED_WHILE_SNAPSHOT_DRAIN_PENDING"
);
assert!(!shutdown.is_finished(), "OWNER_SHUTDOWN_DRAINS_INFLIGHT_SNAPSHOT");
resume.notify_one();
let (owner, result) = tokio::time::timeout(Duration::from_secs(10), shutdown)
.await
.expect("released snapshot lets shutdown finish")
.expect("shutdown task");
result.expect("snapshot work drained");
routes.remove(0);
drop(observer);
drop(owner);
drop(waiters);
let Node {
directory,
writer,
owner: _,
id,
} = node;
drop(writer);
let store = Store::open(directory.path().join("db"), identity(id)).expect("OWNER_SHUTDOWN_SNAPSHOT_RELEASES_LIVE_DATABASE");
assert_eq!(
store
.created_events_page(None, 256)
.expect("retained committed source events")
.events
.len(),
1
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn raft_owner_shutdown_inflight_replication() {
// OpenRaft times append_entries by heartbeat_interval. Keep the real deadline
// longer than readiness checks so only explicit RPC release ends this pause.
let slow = Arc::new(
openraft::Config {
heartbeat_interval: 10_000,
election_timeout_min: 20_000,
election_timeout_max: 30_000,
snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast(1000),
..openraft::Config::default()
}
.validate()
.expect("long real RPC deadline"),
);
let routes = Arc::new(Routes::default());
let mut nodes = Vec::new();
for id in [0, 1, 2] {
nodes.push(Node::start(id, routes.clone(), slow.clone()).await);
}
nodes[0]
.owner()
.bootstrap(members(&[0, 1, 2]))
.await
.expect("real three-node bootstrap");
nodes[0]
.owner()
.raft()
.trigger()
.elect()
.await
.expect("explicit test election readiness");
let mut cluster = Cluster { nodes, routes };
let leader = cluster.leader(&[0, 1, 2]).await;
let response = cluster
.node(leader)
.owner()
.client_write(command(1))
.await
.expect("real replicated write before pause");
cluster.applied(&[0, 1, 2], response.log_id.index).await;
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("replication target");
let entered = Arc::new(tokio::sync::Notify::new());
let resume = Arc::new(tokio::sync::Notify::new());
cluster
.routes
.pause_append
.lock()
.expect("actual request pause")
.insert((leader, follower), (entered.clone(), resume.clone()));
cluster
.node(leader)
.owner()
.raft()
.trigger()
.heartbeat()
.await
.expect("real replication RPC");
tokio::time::timeout(Duration::from_secs(5), entered.notified())
.await
.expect("actual AppendEntries in flight");
let node = cluster.nodes.iter_mut().find(|node| node.id == leader).expect("leader node");
let owner = node.owner.take().expect("leader owner");
let waiters = owner.shutdown_waiters();
let observer = owner.raft().clone();
let shutdown = tokio::spawn(async move {
let result = owner.shutdown().await;
(owner, result)
});
wait_for_shutdown_drain(&waiters).await;
assert!(
observer.metrics().borrow().running_state.is_err(),
"OWNER_CORE_STOPPED_WHILE_REPLICATION_DRAIN_PENDING"
);
assert!(!shutdown.is_finished(), "OWNER_SHUTDOWN_DRAINS_INFLIGHT_REPLICATION");
resume.notify_one();
let (owner, result) = tokio::time::timeout(Duration::from_secs(10), shutdown)
.await
.expect("released real RPC lets shutdown finish")
.expect("replication shutdown task");
result.expect("actual replication reader drained");
cluster.routes.remove(leader);
drop(observer);
drop(owner);
drop(waiters);
let position = cluster
.nodes
.iter()
.position(|node| node.id == leader)
.expect("stopped leader");
let Node {
directory,
writer,
owner: _,
id,
} = cluster.nodes.remove(position);
drop(writer);
let store =
Store::open(directory.path().join("db"), identity(id)).expect("OWNER_SHUTDOWN_REPLICATION_RELEASES_LIVE_DATABASE");
assert_eq!(
store
.created_events_page(None, 256)
.expect("real quorum event remains")
.events
.len(),
1
);
drop(store);
cluster.stop().await;
}
#[path = "committed_created_authorization_tests.rs"]
mod authorization;
@@ -0,0 +1,327 @@
//! Durable OpenRaft application state and selected baseline snapshots.
use crate::{
CaptureRaftEntry, CaptureResponse, CaptureTypeConfig, CheckpointLimits, NativeLogWriter, NativeSnapshotData,
NativeSnapshotError, StoreError, checkpoint_baseline, snapshot,
};
use openraft::{
OptionalSend, RaftSnapshotBuilder, Snapshot, SnapshotMeta, StorageError, StorageIOError, StoredMembership,
storage::RaftStateMachine,
};
use redb::{ReadableDatabase, ReadableTable, ReadableTableMetadata};
use std::{
io::{Seek, Write},
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
pub(crate) const SNAPSHOT_META: &str = "native_snapshot_meta_v1";
type Meta = SnapshotMeta<u64, crate::CaptureNode>;
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct SelectedMeta {
generation: u64,
meta: Meta,
}
pub(crate) fn publish_meta(
meta: &mut redb::Table<&str, &[u8]>,
baseline: &checkpoint_baseline::BaselineMetadata,
value: Meta,
) -> Result<(), StoreError> {
let bytes = serde_json::to_vec(&SelectedMeta {
generation: baseline.generation,
meta: value,
})
.map_err(|_| StoreError::CorruptRecord)?;
if bytes.len() > 1024 * 1024 {
return Err(StoreError::ResourceLimit);
}
meta.insert(SNAPSHOT_META, bytes.as_slice())?;
meta.insert("native_snapshot_meta_required", &[1u8][..])?;
Ok(())
}
pub(crate) fn local_meta(
baseline: &checkpoint_baseline::BaselineMetadata,
identity: crate::NodeIdentity,
) -> Result<Meta, StoreError> {
Ok(Meta {
last_log_id: baseline.header.applied.map(Into::into),
last_membership: StoredMembership::try_from(&baseline.header.membership)?,
snapshot_id: format!("native-local-{:x?}-{}-{}", identity.cluster_id, identity.node_id, baseline.generation),
})
}
pub(crate) fn publish_local_meta(
meta: &mut redb::Table<&str, &[u8]>,
baseline: &checkpoint_baseline::BaselineMetadata,
) -> Result<(), StoreError> {
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
publish_meta(meta, baseline, local_meta(baseline, identity)?)
}
pub(crate) fn selected_meta(
meta: &impl ReadableTable<&'static str, &'static [u8]>,
baseline: &checkpoint_baseline::BaselineMetadata,
) -> Result<Meta, StoreError> {
let Some(bytes) = meta.get(SNAPSHOT_META)? else {
if meta.get("native_snapshot_meta_required")?.is_some() {
return Err(StoreError::CorruptRecord);
}
return local_meta(
baseline,
crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?,
);
};
if bytes.value().len() > 1024 * 1024 {
return Err(StoreError::CorruptRecord);
}
let selected: SelectedMeta = serde_json::from_slice(bytes.value()).map_err(|_| StoreError::CorruptRecord)?;
if selected.generation != baseline.generation
|| selected.meta.last_log_id.map(crate::LogId::from) != baseline.header.applied
|| crate::CaptureMembership::try_from(&selected.meta.last_membership)? != baseline.header.membership
{
return Err(StoreError::CorruptRecord);
}
Ok(selected.meta)
}
#[cfg(test)]
pub(crate) type TestPause = std::sync::Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>;
#[derive(Default)]
pub(crate) struct ReceiveState {
count: AtomicUsize,
#[cfg(test)]
pub(crate) purge_recheck: TestPause,
pub(crate) changed: tokio::sync::Notify,
#[cfg(test)]
pub(crate) waiters: AtomicUsize,
}
pub(crate) struct ReceiveGuard {
state: Arc<ReceiveState>,
}
impl ReceiveGuard {
pub(crate) fn belongs_to(&self, state: &Arc<ReceiveState>) -> bool {
Arc::ptr_eq(&self.state, state)
}
fn new(state: Arc<ReceiveState>) -> Self {
state.count.fetch_add(1, Ordering::AcqRel);
Self { state }
}
}
impl Drop for ReceiveGuard {
fn drop(&mut self) {
self.state.count.fetch_sub(1, Ordering::AcqRel);
self.state.changed.notify_waiters();
}
}
impl ReceiveState {
pub(crate) fn pending(&self) -> bool {
self.count.load(Ordering::Acquire) > 0
}
}
/// Both adapters must use clones of the same writer. Limits apply while receiving.
#[derive(Clone)]
pub struct NativeRaftStateMachine {
pub(crate) writer: NativeLogWriter,
pub(crate) limits: CheckpointLimits,
#[cfg(test)]
pub(crate) build_before_ordering: Arc<TestPause>,
// Declared last: every clone releases its writer before the final owner lease.
pub(crate) owner_lease: Option<Arc<crate::raft_owner::OwnerLease>>,
}
impl NativeRaftStateMachine {
pub fn new(writer: NativeLogWriter, limits: CheckpointLimits) -> Result<Self, NativeSnapshotError> {
limits.validate()?;
limits.snapshot.total_bytes.checked_add(56).ok_or(StoreError::ResourceLimit)?;
Ok(Self {
writer,
limits,
#[cfg(test)]
build_before_ordering: Arc::new(TestPause::default()),
owner_lease: None,
})
}
fn maximum(&self) -> u64 {
self.limits.snapshot.total_bytes + 56
}
async fn current(&self) -> Result<Option<Snapshot<CaptureTypeConfig>>, NativeSnapshotError> {
let owner = self.writer.store.clone();
let limits = self.limits;
tokio::task::spawn_blocking(move || {
let store = owner.lock().map_err(|_| StoreError::CorruptRecord)?;
if !store.writable {
return Err(StoreError::WriteDisabled);
}
let tx = store.database.begin_read()?;
drop(store);
let table = tx.open_table(crate::META)?;
let Some(baseline) = checkpoint_baseline::read_metadata(&table)? else {
return Ok(None);
};
let meta = selected_meta(&table, &baseline)?;
let definitions = [
checkpoint_baseline::HEADS,
checkpoint_baseline::VERSIONS,
checkpoint_baseline::EVENTS,
checkpoint_baseline::OPERATIONS,
];
let mut counts = [0; 4];
for (i, definition) in definitions.iter().enumerate() {
counts[i] = tx.open_table(*definition)?.len()?;
}
let out = snapshot::CapabilityOutput::new(Vec::new(), limits.snapshot.total_bytes);
let mut encoder =
snapshot::Encoder::new(out, &baseline.header, counts, limits.snapshot).map_err(|_| StoreError::CorruptRecord)?;
for (definition, section) in definitions.into_iter().zip([
snapshot::Section::Heads,
snapshot::Section::Versions,
snapshot::Section::Events,
snapshot::Section::Operations,
]) {
for row in tx.open_table(definition)?.iter()? {
let (key, value) = row?;
encoder
.row(snapshot::Row {
section,
key: key.value(),
value: value.value(),
})
.map_err(|_| StoreError::CorruptRecord)?;
}
}
let artifact = encoder.finish().map_err(|_| StoreError::CorruptRecord)?.finish();
let mut data = NativeSnapshotData::new(limits.snapshot.total_bytes + 56).map_err(|_| StoreError::ResourceLimit)?;
data.write_all(&artifact.manifest.bytes)
.map_err(|_| StoreError::ResourceLimit)?;
data.write_all(&artifact.payload).map_err(|_| StoreError::ResourceLimit)?;
data.rewind().map_err(|_| StoreError::CorruptRecord)?;
Ok::<_, StoreError>(Some(Snapshot {
meta,
snapshot: Box::new(data),
}))
})
.await
.map_err(NativeSnapshotError::Worker)?
.map_err(NativeSnapshotError::Store)
}
}
fn failure(error: impl std::error::Error + 'static) -> StorageError<u64> {
StorageIOError::read_state_machine(openraft::AnyError::new(&error)).into()
}
impl RaftStateMachine<CaptureTypeConfig> for NativeRaftStateMachine {
type SnapshotBuilder = Self;
async fn applied_state(
&mut self,
) -> Result<(Option<openraft::LogId<u64>>, StoredMembership<u64, crate::CaptureNode>), StorageError<u64>> {
let (id, membership) = self.writer.applied_state().await.map_err(failure)?;
Ok((id.map(Into::into), StoredMembership::try_from(&membership).map_err(failure)?))
}
async fn apply<I>(&mut self, entries: I) -> Result<Vec<CaptureResponse>, StorageError<u64>>
where
I: IntoIterator<Item = CaptureRaftEntry> + OptionalSend,
I::IntoIter: OptionalSend,
{
self.writer
.apply_committed(entries.into_iter().collect())
.await
.map_err(failure)
}
async fn get_snapshot_builder(&mut self) -> Self {
self.clone()
}
async fn begin_receiving_snapshot(&mut self) -> Result<Box<NativeSnapshotData>, StorageError<u64>> {
let mut data = NativeSnapshotData::new(self.maximum()).map_err(failure)?;
data.attach_receive_guard(ReceiveGuard::new(self.writer.snapshot_receives.clone()));
Ok(Box::new(data))
}
async fn install_snapshot(&mut self, meta: &Meta, data: Box<NativeSnapshotData>) -> Result<(), StorageError<u64>> {
crate::native_snapshot_install::install(self.writer.clone(), self.limits, meta.clone(), data)
.await
.map_err(failure)
}
async fn get_current_snapshot(&mut self) -> Result<Option<Snapshot<CaptureTypeConfig>>, StorageError<u64>> {
self.current().await.map_err(failure)
}
}
impl RaftSnapshotBuilder<CaptureTypeConfig> for NativeRaftStateMachine {
async fn build_snapshot(&mut self) -> Result<Snapshot<CaptureTypeConfig>, StorageError<u64>> {
#[cfg(test)]
{
let hook = self.build_before_ordering.lock().expect("build test hook").take();
if let Some((entered, resume)) = hook {
entered.notify_one();
resume.notified().await;
}
}
let limits = self.limits;
let published_empty = self
.writer
.write(move |store| {
let tx = store.write_transaction()?;
let mut meta = tx.open_table(crate::META)?;
// Decide the empty/nonempty path under the original writer ordering.
if crate::application::read_applied(&meta)?.is_some() {
return Ok(false);
}
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
let membership =
crate::CaptureMembership::decode(meta.get("native_membership")?.ok_or(StoreError::CorruptRecord)?.value())?;
let previous = checkpoint_baseline::read_metadata(&meta)?;
let generation = previous.map_or(Ok(1), |v| v.generation.checked_add(1).ok_or(StoreError::ResourceLimit))?;
let baseline = checkpoint_baseline::BaselineMetadata {
generation,
header: snapshot::Header {
cluster_id: identity.cluster_id,
applied: None,
membership,
binding: crate::application::read_binding(&meta)?,
},
limits,
};
for (live, base) in [
(crate::application::HEADS, checkpoint_baseline::HEADS),
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
] {
if tx.open_table(live)?.len()? != 0 {
return Err(StoreError::CorruptRecord);
}
tx.open_table(base)?.retain(|_, _| false)?;
}
meta.insert("schema", 7u64.to_be_bytes().as_slice())?;
meta.insert(checkpoint_baseline::META_KEY, checkpoint_baseline::encode_metadata(&baseline)?.as_slice())?;
publish_local_meta(&mut meta, &baseline)?;
drop(meta);
store.commit(tx)?;
Ok(true)
})
.await
.map_err(failure)?;
if !published_empty {
self.writer.checkpoint_local(self.limits).await.map_err(failure)?;
}
self.current()
.await
.map_err(failure)?
.ok_or_else(|| failure(StoreError::CorruptRecord))
}
}
#[cfg(test)]
pub(crate) struct Waiter(Arc<ReceiveState>);
#[cfg(test)]
impl Waiter {
pub(crate) fn new(state: Arc<ReceiveState>) -> Self {
state.waiters.fetch_add(1, Ordering::AcqRel);
Self(state)
}
}
#[cfg(test)]
impl Drop for Waiter {
fn drop(&mut self) {
self.0.waiters.fetch_sub(1, Ordering::AcqRel);
}
}
@@ -0,0 +1,583 @@
//! Actual OpenRaft storage traits over real redb; no simulated state machine.
use super::*;
use openraft::{
RaftLogReader, RaftSnapshotBuilder,
storage::{RaftLogStorage, RaftLogStorageExt, RaftStateMachine},
};
use std::{
collections::{BTreeMap, BTreeSet},
io::SeekFrom,
};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
pub(super) type Snapshot = openraft::Snapshot<CaptureTypeConfig>;
pub(super) fn identity(node_id: u64) -> NodeIdentity {
NodeIdentity {
cluster_id: [7; 16],
node_id,
}
}
pub(super) fn binding() -> CaptureBinding {
CaptureBinding {
revision: 9,
binding_id: [8; 16],
target_id: [9; 16],
}
}
pub(super) fn id(index: u64) -> LogId {
LogId {
term: 8,
leader_node: 0,
index,
}
}
pub(super) fn entry(index: u64) -> CaptureRaftEntry {
let payload = if index == 0 {
let membership = CaptureMembership::new(
Some(id(0)),
vec![BTreeSet::from([0])],
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("member"))]),
)
.expect("native membership");
let stored = openraft::StoredMembership::try_from(&membership).expect("stored membership");
openraft::EntryPayload::Membership(stored.membership().clone())
} else {
let byte = u8::try_from(index).expect("bounded test event");
openraft::EntryPayload::Normal(
CaptureCommand::try_created(DecideCreated {
operation_id: [byte; 16],
object: ObjectIdentity {
bucket_incarnation: [2; 16],
key: vec![byte],
},
expected_head: None,
prepared: PreparedIdentity {
version_id: [byte; 16],
preparation_id: [4; 16],
content_digest: [5; 32],
content_length: 123,
},
binding_revision: binding().revision,
})
.expect("native command"),
)
};
CaptureRaftEntry {
log_id: id(index).into(),
payload,
}
}
pub(super) fn entries(end: u64) -> Vec<CaptureRaftEntry> {
(0..=end).map(entry).collect()
}
pub(super) struct Fixture {
pub directory: tempfile::TempDir,
pub writer: NativeLogWriter,
pub logs: NativeRaftLogStore,
pub sm: NativeRaftStateMachine,
pub identity: NodeIdentity,
}
impl Fixture {
pub fn new(node_id: u64) -> Self {
Self::with_identity_binding(identity(node_id), binding())
}
pub fn with_identity_binding(identity: NodeIdentity, binding: CaptureBinding) -> Self {
let directory = tempfile::tempdir().expect("real state machine directory");
let mut store = Store::initialize(directory.path().join("db"), identity).expect("real redb initialization");
store.pin_capture_binding(binding).expect("native binding");
store.enable_native_empty().expect("native mode");
store.promote_created_provenance().expect("provenance");
let writer = NativeLogWriter::new(store);
let logs = NativeRaftLogStore::new(writer.clone());
let sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("real state machine adapter");
Self {
directory,
writer,
logs,
sm,
identity,
}
}
pub async fn apply(&mut self, batch: Vec<CaptureRaftEntry>) -> Vec<CaptureResponse> {
self.logs
.blocking_append(batch.clone())
.await
.expect("actual trait append and library flush callback");
self.sm.apply(batch).await.expect("actual state machine trait apply")
}
pub async fn build(&mut self) -> Snapshot {
self.sm
.get_snapshot_builder()
.await
.build_snapshot()
.await
.expect("actual durable snapshot builder")
}
pub async fn page(&self) -> EventPage {
self.writer
.created_events_page(None, 256)
.await
.expect("authoritative native events")
}
}
pub(super) async fn snapshot_bytes(snapshot: &mut Snapshot) -> Vec<u8> {
snapshot.snapshot.seek(SeekFrom::Start(0)).await.expect("snapshot rewind");
let mut bytes = Vec::new();
snapshot
.snapshot
.read_to_end(&mut bytes)
.await
.expect("actual snapshot bytes");
bytes
}
pub(super) async fn receive(
sm: &mut NativeRaftStateMachine,
bytes: &[u8],
) -> Box<<CaptureTypeConfig as openraft::RaftTypeConfig>::SnapshotData> {
let mut data = sm.begin_receiving_snapshot().await.expect("actual bounded receive handle");
data.write_all(bytes).await.expect("receive real snapshot bytes");
data.seek(SeekFrom::Start(0)).await.expect("received snapshot rewind");
data
}
pub(super) async fn install(sm: &mut NativeRaftStateMachine, snapshot: &mut Snapshot) {
let bytes = snapshot_bytes(snapshot).await;
let data = receive(sm, &bytes).await;
sm.install_snapshot(&snapshot.meta, data)
.await
.expect("actual state machine trait install");
}
#[tokio::test]
async fn raft_state_machine_apply_restart() {
let mut f = Fixture::new(0);
assert_eq!(f.sm.applied_state().await.expect("empty applied state").0, None);
assert!(f.sm.get_current_snapshot().await.expect("no published snapshot").is_none());
let batch = entries(2);
let responses = f.apply(batch.clone()).await;
assert_eq!(responses.len(), 3, "RAFT_SM_REAL_RESPONSE_COUNT");
assert_eq!(responses[0].kind(), CaptureResponseKind::Membership);
assert!(matches!(responses[1].created_result(), Some(CreatedResult::Created { .. })));
let state = f.sm.applied_state().await.expect("durable applied state");
assert_eq!(state.0, Some(id(2).into()));
assert_eq!(state.1.log_id(), &Some(id(0).into()));
let before = f.page().await;
let Fixture {
directory,
writer,
logs,
sm,
identity,
} = f;
drop(sm);
drop(logs);
drop(writer);
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart real store"));
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened state machine");
assert_eq!(
sm.applied_state().await.expect("reopened applied state"),
state,
"RAFT_SM_APPLY_ACK_DURABLE"
);
assert_eq!(writer.created_events_page(None, 256).await.expect("reopened events"), before);
assert_eq!(
sm.apply(batch).await.expect("exact replay responses"),
responses,
"RAFT_SM_REPLAY_SAME_RESPONSES"
);
assert_eq!(
writer
.created_events_page(None, 256)
.await
.expect("no duplicate replay events"),
before
);
}
#[tokio::test]
async fn raft_state_machine_snapshot_build_get_restart() {
let mut f = Fixture::new(0);
f.apply(entries(2)).await;
let mut built = f.build().await;
let meta = built.meta.clone();
let bytes = snapshot_bytes(&mut built).await;
assert_eq!(meta.last_log_id, Some(id(2).into()), "RAFT_SM_SNAPSHOT_META_CUT");
assert_eq!(meta.last_membership, f.sm.applied_state().await.expect("membership").1);
f.apply(vec![entry(3)]).await;
let mut current =
f.sm.get_current_snapshot()
.await
.expect("persisted snapshot")
.expect("snapshot exists");
assert_eq!(current.meta, meta, "RAFT_SM_LIVE_APPLY_DOES_NOT_REPIN_SNAPSHOT");
assert_eq!(snapshot_bytes(&mut current).await, bytes, "RAFT_SM_SNAPSHOT_FROZEN_BYTES");
let Fixture {
directory,
writer,
logs,
sm,
identity,
} = f;
drop(current);
drop(built);
drop(sm);
drop(logs);
drop(writer);
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart snapshot owner"));
let mut sm = NativeRaftStateMachine::new(writer, CheckpointLimits::default()).expect("reopen adapter");
let mut current = sm
.get_current_snapshot()
.await
.expect("restarted current snapshot")
.expect("durable snapshot exists");
assert_eq!(current.meta, meta, "RAFT_SM_COMPLETE_META_RECONSTRUCTED");
assert_eq!(snapshot_bytes(&mut current).await, bytes, "RAFT_SM_RESTART_SNAPSHOT_BYTES");
}
#[tokio::test]
async fn raft_state_machine_receive_bounds() {
let f = Fixture::new(0);
let mut limits = CheckpointLimits::default();
limits.snapshot.total_bytes = 1024;
let mut sm = NativeRaftStateMachine::new(f.writer.clone(), limits).expect("bounded state machine");
let mut data = sm.begin_receiving_snapshot().await.expect("bounded receive");
data.write_all(&[1; 16]).await.expect("small write");
assert!(data.seek(SeekFrom::Start(u64::MAX)).await.is_err(), "RAFT_SM_RECEIVE_SEEK_BOUND");
data.seek(SeekFrom::Start(0)).await.expect("restore seek");
let mut bytes = [0; 16];
data.read_exact(&mut bytes).await.expect("small received bytes");
assert_eq!(bytes, [1; 16], "RAFT_SM_BAD_SEEK_PRESERVES_BYTES");
assert!(data.write_all(&[2; 2048]).await.is_err(), "RAFT_SM_RECEIVE_WRITE_BOUND_BEFORE_INSTALL");
assert_eq!(sm.applied_state().await.expect("receive is not install").0, None);
assert!(sm.get_current_snapshot().await.expect("no received publication").is_none());
}
#[tokio::test]
async fn raft_state_machine_install_rejects_meta_identity() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let bytes = snapshot_bytes(&mut snapshot).await;
let mut target = Fixture::new(1);
target.apply(entries(1)).await;
let old_page = target.page().await;
let stable = std::fs::read(target.directory.path().join("db")).expect("nonempty target bytes");
let mut wrong = snapshot.meta.clone();
wrong.last_log_id = Some(LogId { term: 9, ..id(2) }.into());
let data = receive(&mut target.sm, &bytes).await;
assert!(
target.sm.install_snapshot(&wrong, data).await.is_err(),
"SNAPSHOT_META_BOUND_TO_FRAME: full LogId term cannot differ from snapshot header"
);
assert_eq!(target.sm.applied_state().await.expect("rejection state").0, Some(id(1).into()));
assert_eq!(target.page().await, old_page, "RAFT_SM_META_REJECTION_RETAINS_EVENTS");
assert_eq!(
std::fs::read(target.directory.path().join("db")).expect("unchanged nonempty target"),
stable,
"RAFT_SM_META_REJECTION_NO_MUTATION"
);
assert!(target.sm.get_current_snapshot().await.expect("rejection snapshot").is_none());
let mut wrong = snapshot.meta.clone();
wrong.last_membership = openraft::StoredMembership::default();
let data = receive(&mut target.sm, &bytes).await;
assert!(
target.sm.install_snapshot(&wrong, data).await.is_err(),
"RAFT_SM_MEMBERSHIP_BOUND_TO_FRAME"
);
let mut foreign = Fixture::with_identity_binding(
NodeIdentity {
cluster_id: [6; 16],
..identity(1)
},
binding(),
);
let data = receive(&mut foreign.sm, &bytes).await;
assert!(
foreign.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
"RAFT_SM_CLUSTER_BOUND_TO_DURABLE_OWNER"
);
let mut foreign = Fixture::with_identity_binding(
identity(1),
CaptureBinding {
target_id: [5; 16],
..binding()
},
);
let data = receive(&mut foreign.sm, &bytes).await;
assert!(
foreign.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
"RAFT_SM_CAPTURE_BINDING_BOUND"
);
}
#[tokio::test]
async fn raft_state_machine_install_preserves_local_identity_vote() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
snapshot.meta.snapshot_id = "foreign-snapshot-id-preserved-verbatim".into();
let mut target = Fixture::new(1);
let vote = openraft::Vote::new_committed(10, 1);
target.logs.save_vote(&vote).await.expect("local election vote");
install(&mut target.sm, &mut snapshot).await;
assert_eq!(
target.writer.durable_identity().await.expect("local identity"),
identity(1),
"RAFT_SM_INSTALL_PRESERVES_LOCAL_NODE"
);
assert_eq!(
target.logs.read_vote().await.expect("local vote"),
Some(vote),
"RAFT_SM_INSTALL_PRESERVES_LOCAL_VOTE"
);
let mut current = target
.sm
.get_current_snapshot()
.await
.expect("installed snapshot")
.expect("present");
assert_eq!(current.meta, snapshot.meta, "RAFT_SM_INSTALL_PRESERVES_COMPLETE_INCOMING_META");
assert_eq!(snapshot_bytes(&mut current).await, snapshot_bytes(&mut snapshot).await);
let Fixture {
directory,
writer,
logs,
sm,
identity,
} = target;
drop(current);
drop(sm);
drop(logs);
drop(writer);
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart installed target"));
let mut logs = NativeRaftLogStore::new(writer.clone());
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened installed SM");
assert_eq!(writer.durable_identity().await.expect("restarted local identity"), identity);
assert_eq!(logs.read_vote().await.expect("restarted vote"), Some(vote));
assert_eq!(
sm.get_current_snapshot()
.await
.expect("restarted snapshot")
.expect("present")
.meta,
snapshot.meta,
"RAFT_SM_INCOMING_SNAPSHOT_ID_DURABLE"
);
}
#[tokio::test]
async fn raft_state_machine_install_suffix() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let mut snapshot = source.build().await;
let mut target = Fixture::new(1);
target
.logs
.blocking_append(entries(4))
.await
.expect("retained local log suffix");
target.sm.apply(entries(1)).await.expect("old local applied prefix");
target
.logs
.save_committed(Some(id(4).into()))
.await
.expect("known committed compatible suffix");
install(&mut target.sm, &mut snapshot).await;
assert_eq!(
target
.logs
.get_log_reader()
.await
.try_get_log_entries(3..)
.await
.expect("compatible suffix"),
vec![entry(3), entry(4)],
"RAFT_SM_INSTALL_PRESERVES_COMPATIBLE_COMMITTED_SUFFIX"
);
assert_eq!(
target.logs.read_committed().await.expect("preserved committed marker"),
Some(id(4).into())
);
target
.sm
.apply(vec![entry(3), entry(4)])
.await
.expect("apply retained committed suffix");
assert_eq!(target.page().await.events.len(), 4);
let mut conflict = Fixture::new(2);
let mut batch = entries(4);
for entry in &mut batch[2..] {
entry.log_id = LogId {
term: 9,
..id(entry.log_id.index)
}
.into();
}
conflict
.logs
.blocking_append(batch)
.await
.expect("conflicting uncommitted suffix");
conflict.sm.apply(entries(1)).await.expect("protected applied prefix");
install(&mut conflict.sm, &mut snapshot).await;
assert!(
conflict
.logs
.get_log_reader()
.await
.try_get_log_entries(3..)
.await
.expect("discarded conflict")
.is_empty(),
"RAFT_SM_INSTALL_DROPS_ONLY_UNCOMMITTED_CONFLICT"
);
conflict.apply(vec![entry(3)]).await;
assert_eq!(conflict.sm.applied_state().await.expect("new suffix applied").0, Some(id(3).into()));
}
#[tokio::test]
async fn raft_state_machine_install_restart_continue() {
let mut source = Fixture::new(0);
source.apply(entries(2)).await;
let expected = source.page().await;
let mut snapshot = source.build().await;
let mut target = Fixture::new(1);
install(&mut target.sm, &mut snapshot).await;
assert_eq!(target.page().await, expected, "RAFT_SM_INSTALL_REBUILDS_SOURCE_EVENTS");
let Fixture {
directory,
writer,
logs,
sm,
identity,
} = target;
drop(sm);
drop(logs);
drop(writer);
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart installed redb"));
let mut logs = NativeRaftLogStore::new(writer.clone());
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("restart installed adapter");
assert_eq!(sm.applied_state().await.expect("snapshot applied state").0, Some(id(2).into()));
logs.blocking_append(vec![entry(3)])
.await
.expect("append after install restart");
sm.apply(vec![entry(3)]).await.expect("apply after install restart");
let all = writer.created_events_page(None, 256).await.expect("retained plus new event");
assert_eq!(all.events.len(), 3);
assert_eq!(all.events[..2], expected.events);
let next = writer
.created_events_page(expected.next, 1)
.await
.expect("snapshot cursor resumes");
assert_eq!(next.events[0].event_id, [3; 16], "RAFT_SM_INSTALL_CURSOR_CONTINUITY");
}
#[tokio::test]
async fn raft_state_machine_empty_snapshot_restart_continue() {
let mut source = Fixture::new(0);
let mut target = Fixture::new(42);
let mut snapshot = source.sm.build_snapshot().await.expect("durable empty baseline");
assert_eq!(snapshot.meta.last_log_id, None);
let empty_meta = snapshot.meta.clone();
let received = receive(&mut target.sm, &snapshot_bytes(&mut snapshot).await).await;
target
.sm
.install_snapshot(&empty_meta, received)
.await
.expect("install empty frame");
let mut target = target.reopen();
let current = target
.sm
.get_current_snapshot()
.await
.expect("empty after reopen")
.expect("durable selection");
assert_eq!(current.meta, empty_meta);
target
.logs
.blocking_append(entries(1))
.await
.expect("real append after empty install");
target.sm.apply(entries(1)).await.expect("real apply after empty install");
let mut target = target.reopen();
assert_eq!(
target.sm.applied_state().await.expect("replay empty baseline + suffix").0,
Some(id(1).into())
);
assert_eq!(
target
.writer
.created_events_page(None, 16)
.await
.expect("event after empty baseline")
.events
.len(),
1
);
let refreshed = target
.sm
.build_snapshot()
.await
.expect("nonempty checkpoint supersedes empty");
assert_eq!(refreshed.meta.last_log_id, Some(id(1).into()));
}
impl Fixture {
fn reopen(self) -> Self {
let Self {
directory,
writer,
logs,
sm,
identity,
} = self;
drop(sm);
drop(logs);
drop(writer);
let writer =
NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("real empty baseline restart"));
let logs = NativeRaftLogStore::new(writer.clone());
let sm =
NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened empty baseline adapter");
Self {
directory,
writer,
logs,
sm,
identity,
}
}
}
#[tokio::test]
async fn raft_state_machine_build_competes_with_initial_apply() {
let mut target = Fixture::new(0);
target
.logs
.blocking_append(entries(1))
.await
.expect("real pending initial batch");
let entered = std::sync::Arc::new(tokio::sync::Notify::new());
let resume = std::sync::Arc::new(tokio::sync::Notify::new());
*target.sm.build_before_ordering.lock().expect("build pause") = Some((entered.clone(), resume.clone()));
let mut builder = target.sm.get_snapshot_builder().await;
let task = tokio::spawn(async move { builder.build_snapshot().await });
tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified())
.await
.expect("builder began before initial apply");
target
.sm
.apply(entries(1))
.await
.expect("actual initial apply while builder pending");
resume.notify_one();
let snapshot = tokio::time::timeout(std::time::Duration::from_secs(5), task)
.await
.expect("build continues")
.expect("builder task")
.expect("initial apply does not make snapshot build a fatal storage error");
assert_eq!(
snapshot.meta.last_log_id,
Some(id(1).into()),
"SNAPSHOT_BUILD_INITIAL_APPLY_RACE_USES_NONEMPTY_CHECKPOINT"
);
}
+2 -2
View File
@@ -136,7 +136,7 @@ openraft::declare_raft_types!(
NodeId = u64,
Node = CaptureNode,
Entry = openraft::Entry<Self>,
SnapshotData = std::io::Cursor<Vec<u8>>,
SnapshotData = crate::NativeSnapshotData,
AsyncRuntime = openraft::TokioRuntime,
Responder = openraft::impls::OneshotResponder<Self>,
);
@@ -925,7 +925,7 @@ mod tests {
NodeId = u64,
Node = CaptureNode,
Entry = CaptureRaftEntry,
SnapshotData = std::io::Cursor<Vec<u8>>,
SnapshotData = crate::NativeSnapshotData,
AsyncRuntime = openraft::TokioRuntime,
Responder = openraft::impls::OneshotResponder<CaptureTypeConfig>,
>,
+155
View File
@@ -38,6 +38,161 @@ pub enum SnapshotError {
#[error("snapshot IO failed")]
Io(#[from] std::io::Error),
}
/// Fixed capability declaration. Consumers validate these untrusted bytes before decoding.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CapabilityManifest {
pub bytes: [u8; 56],
}
impl CapabilityManifest {
pub fn encode(&self) -> [u8; 56] {
self.bytes
}
pub fn decode(bytes: &[u8]) -> Result<Self, SnapshotError> {
format(bytes.len() == 56, "capability length")?;
format(&bytes[..6] == b"CCAP\x01\x01", "capability version/framing")?;
format(bytes[6..14] == 6u64.to_be_bytes(), "capability schema")?;
format(bytes[14..16] == [1, 1], "operation codec")?;
let mut encoded = [0; 56];
encoded.copy_from_slice(bytes);
Ok(Self { bytes: encoded })
}
fn payload_length(&self) -> u64 {
let mut bytes = [0; 8];
bytes.copy_from_slice(&self.bytes[16..24]);
u64::from_be_bytes(bytes)
}
}
/// Manifest and payload form one explicit input. This does not grant installation authority.
pub struct CapabilityArtifact<T> {
pub manifest: CapabilityManifest,
pub payload: T,
}
pub(crate) struct CapabilityOutput<W> {
io: W,
hash: Sha256,
used: u64,
max: u64,
}
impl<W: Write> CapabilityOutput<W> {
pub(crate) fn new(io: W, max: u64) -> Self {
Self {
io,
hash: Sha256::new(),
used: 0,
max,
}
}
pub(crate) fn finish(self) -> CapabilityArtifact<W> {
let mut bytes = [0; 56];
bytes[..6].copy_from_slice(b"CCAP\x01\x01");
bytes[6..14].copy_from_slice(&6u64.to_be_bytes());
bytes[14..16].copy_from_slice(&[1, 1]);
bytes[16..24].copy_from_slice(&self.used.to_be_bytes());
bytes[24..].copy_from_slice(&self.hash.finalize());
CapabilityArtifact {
manifest: CapabilityManifest { bytes },
payload: self.io,
}
}
}
impl<W: Write> Write for CapabilityOutput<W> {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
let requested = u64::try_from(bytes.len()).map_err(std::io::Error::other)?;
if self.used.checked_add(requested).is_none_or(|n| n > self.max) {
return Err(std::io::Error::other("capability payload limit"));
}
let written = self.io.write(bytes)?;
let part = bytes
.get(..written)
.ok_or_else(|| std::io::Error::other("invalid writer count"))?;
let count = u64::try_from(written).map_err(std::io::Error::other)?;
self.used = self
.used
.checked_add(count)
.ok_or_else(|| std::io::Error::other("capability length overflow"))?;
self.hash.update(part);
Ok(written)
}
fn flush(&mut self) -> std::io::Result<()> {
self.io.flush()
}
}
struct CapabilityInput<R> {
io: R,
hash: Sha256,
used: u64,
max: u64,
}
impl<R: Read> Read for CapabilityInput<R> {
fn read(&mut self, bytes: &mut [u8]) -> std::io::Result<usize> {
// At the boundary, permit only the existing decoder's one-byte EOF probe.
let remaining = self
.max
.checked_sub(self.used)
.ok_or_else(|| std::io::Error::other("capability length overflow"))?;
let requested = u64::try_from(bytes.len()).map_err(std::io::Error::other)?;
let allowed = requested.min(if remaining == 0 { 1 } else { remaining });
let allowed = usize::try_from(allowed).map_err(std::io::Error::other)?;
let read = self.io.read(&mut bytes[..allowed])?;
let part = bytes
.get(..read)
.ok_or_else(|| std::io::Error::other("invalid reader count"))?;
let count = u64::try_from(read).map_err(std::io::Error::other)?;
let used = self
.used
.checked_add(count)
.ok_or_else(|| std::io::Error::other("capability length overflow"))?;
if used > self.max {
return Err(std::io::Error::other("capability payload limit"));
}
self.used = used;
self.hash.update(part);
Ok(read)
}
}
struct CapabilitySink<'a, S>(&'a mut S);
impl<S: StagingSink> StagingSink for CapabilitySink<'_, S> {
fn row(&mut self, section: Section, key: &[u8], value: &[u8]) -> std::io::Result<()> {
if section == Section::Operations {
crate::application::validate_provenance_encoding(value).map_err(std::io::Error::other)?;
}
self.0.row(section, key, value)
}
}
/// Validates framing, declared capability and operation encoding into isolated staging.
/// Failure may leave partial staging rows, which the caller must discard.
/// Success does not prove cross-table consistency, historical truth or source authority.
pub fn decode_capability_artifact(
artifact: CapabilityArtifact<impl Read>,
sink: &mut impl StagingSink,
limits: Limits,
) -> Result<Header, SnapshotError> {
let manifest = CapabilityManifest::decode(&artifact.manifest.bytes)?;
let length = manifest.payload_length();
limit(length <= limits.total_bytes, "capability payload")?;
let mut reader = CapabilityInput {
io: artifact.payload,
hash: Sha256::new(),
used: 0,
max: length,
};
let header = decode(&mut reader, &mut CapabilitySink(sink), limits)?;
format(reader.used == length, "capability payload length")?;
if reader.hash.finalize().as_slice() != &manifest.bytes[24..] {
return Err(SnapshotError::Digest);
}
Ok(header)
}
/// Row values remain opaque. Operation values will contain the canonical decision
/// LogId followed by the existing canonical operation bytes, validated by callers.
pub struct Row<'a> {
+559
View File
@@ -0,0 +1,559 @@
//! Disposable event ordering index. EVENTS remains the sole event authority.
use crate::{
CaptureBinding, CreatedEvent, CreatedResult, LogId, NativeLogWriter, NativeWriteError, Store, StoreError, application,
decode_id, encode_id,
};
use redb::{ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition, WriteTransaction};
pub const MAX_EVENT_PAGE: usize = 256;
pub(crate) const EVENT_SEQUENCE: TableDefinition<u64, &[u8]> = TableDefinition::new("event_sequence_cache_v1");
/// Persist only after every event preceding this cursor has a durable consumer ACK.
/// Deserialization does not grant authority: each use checks the stored event.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct EventCursor {
cluster: [u8; 16],
term: u64,
leader: u64,
index: u64,
event: [u8; 16],
binding_id: [u8; 16],
target_id: [u8; 16],
revision: u64,
}
impl EventCursor {
pub(crate) fn new(cluster: [u8; 16], event: [u8; 16], value: &CreatedEvent) -> Self {
Self {
cluster,
term: value.decision_log_id.term,
leader: value.decision_log_id.leader_node,
index: value.decision_log_id.index,
event,
binding_id: value.binding.binding_id,
target_id: value.binding.target_id,
revision: value.binding.revision,
}
}
fn matches(&self, cluster: [u8; 16], event: [u8; 16], value: &CreatedEvent) -> bool {
self == &Self::new(cluster, event, value)
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SourceEvent {
pub event_id: [u8; 16],
pub captured: CreatedEvent,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct EventPage {
pub events: Vec<SourceEvent>,
pub next: Option<EventCursor>,
}
fn encoded_index(id: LogId, event: [u8; 16]) -> Vec<u8> {
let mut bytes = encode_id(id).to_vec();
bytes.extend(event);
bytes
}
fn decode_index(bytes: &[u8]) -> Result<(LogId, [u8; 16]), StoreError> {
if bytes.len() != 40 {
return Err(StoreError::CorruptRecord);
}
let id = decode_id(&bytes[..24])?;
let event: [u8; 16] = bytes[24..].try_into().map_err(|_| StoreError::CorruptRecord)?;
if event == [0; 16] {
return Err(StoreError::CorruptRecord);
}
Ok((id, event))
}
pub(crate) fn insert_event_index(tx: &WriteTransaction, id: LogId, event: [u8; 16]) -> Result<(), StoreError> {
let bytes = encoded_index(id, event);
let mut index = tx.open_table(EVENT_SEQUENCE)?;
if index.get(id.index)?.is_some() {
return Err(StoreError::CorruptRecord);
}
index.insert(id.index, bytes.as_slice())?;
Ok(())
}
/// Check the event semantics required to derive ordering from checkpoint authority.
/// This deliberately does not grant snapshot installation or source authority.
fn validated_event(
key: &[u8],
bytes: &[u8],
operations: &impl ReadableTable<&'static [u8], &'static [u8]>,
versions: &impl ReadableTable<&'static [u8], &'static [u8]>,
schema: u64,
binding: &CaptureBinding,
applied: Option<LogId>,
) -> Result<([u8; 16], CreatedEvent), StoreError> {
let event_id: [u8; 16] = key.try_into().map_err(|_| StoreError::CorruptRecord)?;
let event = application::decode_event(bytes)?;
let cut = applied.ok_or(StoreError::CorruptRecord)?;
let operation = operations.get(key)?.ok_or(StoreError::CorruptRecord)?;
let first_decision = if matches!(schema, 5 | 6 | 7) {
Some(application::decode_provenance(operation.value())?.first_decision_log_id)
} else {
None
};
let operation = application::decode_stored_operation(operation.value(), schema)?;
let command = crate::DecideCreated::decode(&operation.canonical_command)?;
let version_key = application::version_key(&event.object, event.prepared.version_id)?;
let version = versions.get(version_key.as_slice())?.ok_or(StoreError::CorruptRecord)?;
let version = application::decode_version(version.value())?;
if first_decision.is_some_and(|first| first != event.decision_log_id)
|| event_id == [0; 16]
|| event.decision_log_id.index > cut.index
|| (event.decision_log_id.index == cut.index && event.decision_log_id != cut)
|| event.binding != *binding
|| command.operation_id != event_id
|| command.object != event.object
|| command.prepared != event.prepared
|| command.binding_revision != event.binding.revision
|| operation.result
!= (CreatedResult::Created {
version_id: event.prepared.version_id,
event_id,
decision_log_id: event.decision_log_id,
})
|| version
!= (crate::CreatedVersion {
prepared: event.prepared.clone(),
decision_log_id: event.decision_log_id,
binding_revision: event.binding.revision,
})
{
return Err(StoreError::CorruptRecord);
}
Ok((event_id, event))
}
/// Rebuild from the same authority rows emitted by the existing four-section checkpoint.
/// Callers must independently establish authority and validate all other state before promotion.
pub(crate) fn rebuild_from_authority(
tx: &WriteTransaction,
schema: u64,
binding: &CaptureBinding,
applied: Option<LogId>,
) -> Result<(), StoreError> {
let events = tx.open_table(application::EVENTS)?;
let operations = tx.open_table(application::OPERATIONS)?;
let versions = tx.open_table(application::VERSIONS)?;
{
let mut index = tx.open_table(EVENT_SEQUENCE)?;
index.retain(|_, _| false)?;
}
for row in events.iter()? {
let (key, bytes) = row?;
let (event, value) = validated_event(key.value(), bytes.value(), &operations, &versions, schema, binding, applied)?;
insert_event_index(tx, value.decision_log_id, event)?;
}
Ok(())
}
impl Store {
pub(crate) fn ensure_event_index(&mut self) -> Result<(), StoreError> {
let read = self.database.begin_read()?;
let meta = read.open_table(crate::META)?;
let schema = crate::number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
if !matches!(schema, 3..=7) {
return Ok(());
}
let binding = application::read_binding(&meta)?;
let applied = application::read_applied(&meta)?;
let events = read.open_table(application::EVENTS)?;
let complete = match read.open_table(EVENT_SEQUENCE) {
Ok(index) => {
let mut complete = index.len()? == events.len()?;
if complete {
for row in events.iter()? {
let (key, bytes) = row?;
let value = application::decode_event(bytes.value())?;
let expected =
encoded_index(value.decision_log_id, key.value().try_into().map_err(|_| StoreError::CorruptRecord)?);
if index
.get(value.decision_log_id.index)?
.is_none_or(|stored| stored.value() != expected.as_slice())
{
complete = false;
break;
}
}
}
complete
}
Err(redb::TableError::TableDoesNotExist(_)) => false,
Err(error) => return Err(error.into()),
};
drop(events);
drop(meta);
drop(read);
if !complete {
let tx = self.write_transaction()?;
rebuild_from_authority(&tx, schema, &binding, applied)?;
self.commit(tx)?;
}
Ok(())
}
/// Read a bounded page without changing any delivery ACK or cursor state.
pub fn created_events_page(&self, cursor: Option<&EventCursor>, limit: usize) -> Result<EventPage, StoreError> {
if !self.writable {
return Err(StoreError::WriteDisabled);
}
if limit == 0 || limit > MAX_EVENT_PAGE {
return Err(StoreError::InvalidEventPage);
}
let tx = self.database.begin_read()?;
let meta = tx.open_table(crate::META)?;
let schema = crate::number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
if !matches!(schema, 3..=7) {
return Err(StoreError::UnsupportedSchema);
}
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
let events = tx.open_table(application::EVENTS)?;
let index = tx.open_table(EVENT_SEQUENCE)?;
if index.len()? != events.len()? {
return Err(StoreError::CorruptRecord);
}
let read_event = |id: u64, bytes: &[u8]| -> Result<SourceEvent, StoreError> {
let (log, event_id) = decode_index(bytes)?;
let value = events.get(event_id.as_slice())?.ok_or(StoreError::CorruptRecord)?;
let captured = application::decode_event(value.value())?;
if log.index != id || captured.decision_log_id != log {
return Err(StoreError::CorruptRecord);
}
Ok(SourceEvent { event_id, captured })
};
if let Some(cursor) = cursor {
let stored = index.get(cursor.index)?.ok_or(StoreError::IdentityMismatch)?;
let event = read_event(cursor.index, stored.value())?;
if !cursor.matches(identity.cluster_id, event.event_id, &event.captured) {
return Err(StoreError::IdentityMismatch);
}
}
let start = cursor.map_or(0, |c| c.index);
let range = if cursor.is_some() {
(std::ops::Bound::Excluded(start), std::ops::Bound::Unbounded)
} else {
(std::ops::Bound::Included(start), std::ops::Bound::Unbounded)
};
let mut page = EventPage {
events: Vec::with_capacity(limit),
next: cursor.cloned(),
};
for row in index.range(range)?.take(limit) {
let (id, bytes) = row?;
let event = read_event(id.value(), bytes.value())?;
page.next = Some(EventCursor::new(identity.cluster_id, event.event_id, &event.captured));
page.events.push(event);
}
Ok(page)
}
}
impl NativeLogWriter {
pub async fn created_events_page(&self, cursor: Option<EventCursor>, limit: usize) -> Result<EventPage, NativeWriteError> {
let owner = self.store.clone();
tokio::task::spawn_blocking(move || {
owner
.lock()
.map_err(|_| NativeWriteError::Poisoned)?
.created_events_page(cursor.as_ref(), limit)
.map_err(NativeWriteError::Store)
})
.await
.map_err(NativeWriteError::Worker)?
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{CaptureCommand, CaptureRaftEntry, DecideCreated, NodeIdentity, ObjectIdentity, PreparedIdentity, snapshot};
use std::io;
fn identity() -> NodeIdentity {
NodeIdentity {
cluster_id: [7; 16],
node_id: 0,
}
}
fn binding() -> CaptureBinding {
CaptureBinding {
revision: 9,
binding_id: [8; 16],
target_id: [9; 16],
}
}
fn id(index: u64) -> LogId {
LogId {
term: 8,
leader_node: 0,
index,
}
}
fn command(event: u8) -> DecideCreated {
DecideCreated {
operation_id: [event; 16],
object: ObjectIdentity {
bucket_incarnation: [2; 16],
key: vec![event],
},
expected_head: None,
prepared: PreparedIdentity {
version_id: [event; 16],
preparation_id: [4; 16],
content_digest: [5; 32],
content_length: 123,
},
binding_revision: 9,
}
}
fn apply(store: &mut Store, index: u64, command: DecideCreated) -> CreatedResult {
store
.append_native(&[CaptureRaftEntry {
log_id: id(index).into(),
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("command")),
}])
.expect("append native");
store.save_committed(id(index)).expect("commit native");
store
.apply_next_native(id(index))
.expect("apply native")
.created_result()
.expect("created response")
.clone()
}
fn fixture() -> (tempfile::TempDir, Store) {
let dir = tempfile::tempdir().expect("directory");
let mut store = Store::initialize(dir.path().join("source.redb"), identity()).expect("initialize");
store.pin_capture_binding(binding()).expect("pin");
store.enable_native_empty().expect("native");
apply(&mut store, 0, command(9));
apply(&mut store, 1, command(7));
(dir, store)
}
#[test]
fn committed_events_page_in_decision_order() {
let (_dir, store) = fixture();
let first = store.created_events_page(None, 1).expect("first");
assert_eq!(
first.events.iter().map(|e| e.event_id).collect::<Vec<_>>(),
vec![[9; 16]],
"SOURCE_PAGE_ORDER_BOUND"
);
let next = store.created_events_page(first.next.as_ref(), 1).expect("second");
assert_eq!(next.events[0].event_id, [7; 16]);
assert_eq!(next.events[0].captured.binding, binding());
assert!(
store
.created_events_page(next.next.as_ref(), 1)
.expect("end")
.events
.is_empty()
);
}
#[test]
fn replay_and_rejections_do_not_duplicate_events() {
let (_dir, mut store) = fixture();
apply(&mut store, 2, command(9));
let mut rejected = command(3);
rejected.object = command(9).object;
assert!(matches!(apply(&mut store, 3, rejected), CreatedResult::HeadMismatch { .. }));
let page = store.created_events_page(None, 256).expect("page");
assert_eq!(page.events.len(), 2, "SOURCE_PAGE_REPLAY_BOUND");
}
#[test]
fn reopen_rebuilds_missing_or_stale_index() {
let (dir, mut store) = fixture();
apply(&mut store, 2, command(1));
let expected = store.created_events_page(None, 256).expect("before");
let tx = store.write_transaction().expect("remove cache");
tx.delete_table(EVENT_SEQUENCE).expect("delete cache");
store.commit(tx).expect("cache deletion");
drop(store);
let mut store = Store::open(dir.path().join("source.redb"), identity()).expect("rebuild missing");
assert_eq!(store.created_events_page(None, 256).expect("rebuilt"), expected);
let tx = store.write_transaction().expect("stale cache");
{
let mut index = tx.open_table(EVENT_SEQUENCE).expect("cache");
index.remove(1).expect("omit middle item");
index.insert(200, b"bad".as_slice()).expect("same-count stale item");
}
store.commit(tx).expect("cache damage");
drop(store);
let store = Store::open(dir.path().join("source.redb"), identity()).expect("rebuild stale");
assert_eq!(store.created_events_page(None, 256).expect("rebuilt stale"), expected);
}
#[test]
fn cursor_binds_cluster_term_leader_and_event() {
let (_dir, store) = fixture();
let cursor = store.created_events_page(None, 1).expect("page").next.expect("cursor");
let mut bad = Vec::new();
let mut c = cursor.clone();
c.cluster = [6; 16];
bad.push(c);
let mut c = cursor.clone();
c.term += 1;
bad.push(c);
let mut c = cursor.clone();
c.leader += 1;
bad.push(c);
let mut c = cursor.clone();
c.event = [7; 16];
bad.push(c);
let mut c = cursor.clone();
c.revision += 1;
bad.push(c);
let mut c = cursor.clone();
c.binding_id = [6; 16];
bad.push(c);
let mut c = cursor.clone();
c.target_id = [6; 16];
bad.push(c);
for cursor in bad {
assert_eq!(
store.created_events_page(Some(&cursor), 1).err().map(|e| e.code()),
Some("identity_mismatch"),
"SOURCE_PAGE_CURSOR_BOUND"
);
}
let encoded = serde_json::to_vec(&cursor).expect("cursor bytes");
let decoded: EventCursor = serde_json::from_slice(&encoded).expect("cursor decode");
assert_eq!(store.created_events_page(Some(&decoded), 1).expect("resume").events[0].event_id, [7; 16]);
}
#[test]
fn page_is_bounded_and_new_lower_uuid_is_not_lost() {
let (_dir, mut store) = fixture();
let page = store.created_events_page(None, 1).expect("one");
assert_eq!(page.events.len(), 1);
apply(&mut store, 2, command(1));
let next = store.created_events_page(page.next.as_ref(), 256).expect("later");
assert_eq!(next.events.iter().map(|e| e.event_id).collect::<Vec<_>>(), vec![[7; 16], [1; 16]]);
for limit in [0, MAX_EVENT_PAGE + 1, usize::MAX] {
assert_eq!(store.created_events_page(None, limit).err().map(|e| e.code()), Some("invalid_event_page"));
}
store.writable = false;
assert_eq!(store.created_events_page(None, 1).err().map(|e| e.code()), Some("write_disabled"));
store.writable = true;
let mut max = page.next.expect("cursor");
max.index = u64::MAX;
assert_eq!(
store.created_events_page(Some(&max), 1).err().map(|e| e.code()),
Some("identity_mismatch")
);
}
struct Staging<'a>(&'a WriteTransaction);
impl snapshot::StagingSink for Staging<'_> {
fn row(&mut self, section: snapshot::Section, key: &[u8], value: &[u8]) -> io::Result<()> {
let table = match section {
snapshot::Section::Heads => application::HEADS,
snapshot::Section::Versions => application::VERSIONS,
snapshot::Section::Events => application::EVENTS,
snapshot::Section::Operations => application::OPERATIONS,
};
self.0
.open_table(table)
.map_err(io::Error::other)?
.insert(key, value)
.map_err(io::Error::other)?;
Ok(())
}
}
fn limits() -> snapshot::Limits {
snapshot::Limits {
total_bytes: 1_048_576,
membership_bytes: 4096,
key_bytes: 4096,
value_bytes: 8192,
records: 1024,
}
}
async fn artifact(store: Store) -> (snapshot::CapabilityArtifact<Vec<u8>>, EventPage) {
let writer = NativeLogWriter::new(store);
let before = writer.created_events_page(None, 256).await.expect("async page");
let artifact = writer
.snapshot_view()
.await
.expect("snapshot view")
.stream_capability_artifact(Vec::new(), limits())
.expect("capability export");
(artifact, before)
}
fn stage(artifact: snapshot::CapabilityArtifact<Vec<u8>>, damage: bool) -> Result<(tempfile::TempDir, Store), StoreError> {
let dir = tempfile::tempdir().expect("stage directory");
let database = redb::Database::create(dir.path().join("stage.redb")).expect("isolated database");
let tx = database.begin_write()?;
let header = snapshot::decode_capability_artifact(
snapshot::CapabilityArtifact {
manifest: artifact.manifest,
payload: std::io::Cursor::new(artifact.payload),
},
&mut Staging(&tx),
limits(),
)
.expect("real capability decoder");
if damage {
let mut events = tx.open_table(application::EVENTS)?;
let value = events.get([9; 16].as_slice())?.expect("event").value().to_vec();
events.insert([6; 16].as_slice(), value.as_slice())?;
events.remove([9; 16].as_slice())?;
}
rebuild_from_authority(&tx, 6, &header.binding, header.applied)?;
{
let mut meta = tx.open_table(crate::META)?;
meta.insert("schema", 6u64.to_be_bytes().as_slice())?;
meta.insert(
"identity",
crate::encode_identity(NodeIdentity {
cluster_id: header.cluster_id,
..identity()
})
.as_slice(),
)?;
meta.insert("capture_binding", crate::created::binding_bytes(&header.binding)?.as_slice())?;
let mut applied = vec![1];
applied.extend(encode_id(header.applied.expect("cut")));
meta.insert("last_applied", applied.as_slice())?;
}
tx.commit().expect("stage commit");
// A test-only staging handle, not a production installer or authority grant.
Ok((
dir,
Store {
database,
writable: true,
},
))
}
#[tokio::test]
async fn snapshot_authority_rebuild_matches_source() {
let (_dir, mut store) = fixture();
apply(&mut store, 2, command(9));
let mut rejected = command(3);
rejected.object = command(9).object;
apply(&mut store, 3, rejected);
store.promote_created_provenance().expect("schema6");
let (artifact, before) = artifact(store).await;
let (_stage, restored) = stage(artifact, false).expect("staged authority index");
assert_eq!(
restored.created_events_page(None, 256).expect("restored page"),
before,
"SOURCE_PAGE_SNAPSHOT_BOUND"
);
let first = restored.created_events_page(None, 1).expect("first");
assert_eq!(
restored.created_events_page(first.next.as_ref(), 1).expect("resume").events[0],
before.events[1]
);
}
#[tokio::test]
async fn snapshot_event_operation_mismatch_rejected() {
let (_dir, mut store) = fixture();
store.promote_created_provenance().expect("schema6");
let (artifact, _) = artifact(store).await;
assert_eq!(
stage(artifact, true).err().map(|e| e.code()),
Some("corrupt_record"),
"SOURCE_PAGE_SNAPSHOT_BOUND"
);
}
}
@@ -0,0 +1,451 @@
//! Same-UID local reads from an owner's live native store. No ACK or cursor writes.
use crate::{EventCursor, EventPage, NativeLogWriter, NativeWriteError, NodeIdentity, StoreError};
use redb::ReadableDatabase;
use serde::{Deserialize, Serialize};
use std::{
fs, io,
os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt},
path::{Path, PathBuf},
time::Duration,
};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::{UnixListener, UnixStream},
sync::{OwnedSemaphorePermit, watch},
task::JoinSet,
};
pub const WIRE_VERSION: u16 = 1;
pub const MAX_REQUEST_BYTES: usize = 4096;
pub const MAX_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
pub const MAX_CONNECTIONS: usize = 4;
pub const IO_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReadPage {
pub version: u16,
pub expected_cluster: [u8; 16],
pub cursor: Option<EventCursor>,
pub limit: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReadPageResponse {
pub version: u16,
pub identity: NodeIdentity,
pub page: EventPage,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum Rejection {
InvalidRequest,
WrongCluster,
InvalidCursor,
StoreUnavailable,
ResponseLimit,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "status", deny_unknown_fields)]
pub enum ReadPageReply {
Page { response: ReadPageResponse },
Rejected { version: u16, error: Rejection },
}
#[derive(Debug, thiserror::Error)]
pub enum ReaderError {
#[error("SOURCE_READER_IO")]
Io(#[from] io::Error),
#[error("SOURCE_READER_PERMISSION")]
Permission,
#[error("SOURCE_READER_PATH_OCCUPIED")]
Occupied,
#[error("SOURCE_READER_PATH_CHANGED")]
PathChanged,
#[error("SOURCE_READER_TIMEOUT")]
Timeout,
#[error("SOURCE_READER_PROTOCOL")]
Protocol,
#[error("SOURCE_READER_REJECTED: {0:?}")]
Rejected(Rejection),
#[error("SOURCE_READER_WORKER")]
Worker(#[source] tokio::task::JoinError),
}
impl crate::Store {
fn durable_source_identity(&self) -> Result<NodeIdentity, StoreError> {
let tx = self.database.begin_read()?;
let meta = tx.open_table(crate::META)?;
let identity = meta.get("identity")?.ok_or(StoreError::CorruptRecord)?;
crate::decode_identity(identity.value())
}
}
impl NativeLogWriter {
/// Read the validated durable identity, never an owner-supplied cluster label.
pub async fn durable_identity(&self) -> Result<NodeIdentity, NativeWriteError> {
let owner = self.store.clone();
tokio::task::spawn_blocking(move || {
owner
.lock()
.map_err(|_| NativeWriteError::Poisoned)?
.durable_source_identity()
.map_err(NativeWriteError::Store)
})
.await
.map_err(NativeWriteError::Worker)?
}
}
fn uid() -> u32 {
rustix::process::geteuid().as_raw()
}
fn fingerprint(meta: &fs::Metadata) -> (u64, u64) {
(meta.dev(), meta.ino())
}
fn private_parent(path: &Path) -> Result<(PathBuf, (u64, u64)), ReaderError> {
if !path.is_absolute() || path.file_name().is_none() {
return Err(ReaderError::Permission);
}
let parent = path.parent().ok_or(ReaderError::Permission)?;
let meta = fs::symlink_metadata(parent)?;
if !meta.is_dir() || meta.uid() != uid() || meta.mode() & 0o077 != 0 {
return Err(ReaderError::Permission);
}
let resolved = fs::canonicalize(parent)?;
let resolved_meta = fs::symlink_metadata(&resolved)?;
if fingerprint(&meta) != fingerprint(&resolved_meta) {
return Err(ReaderError::PathChanged);
}
Ok((resolved.join(path.file_name().ok_or(ReaderError::Permission)?), fingerprint(&meta)))
}
fn owned_socket(path: &Path) -> Result<fs::Metadata, ReaderError> {
let meta = fs::symlink_metadata(path)?;
if !meta.file_type().is_socket() || meta.uid() != uid() {
return Err(ReaderError::Permission);
}
Ok(meta)
}
fn socket_metadata(path: &Path) -> Result<(u64, u64), ReaderError> {
let meta = owned_socket(path)?;
if meta.mode() & 0o077 != 0 {
return Err(ReaderError::Permission);
}
Ok(fingerprint(&meta))
}
fn peer(stream: &UnixStream) -> Result<(), ReaderError> {
if stream.peer_cred()?.uid() != uid() {
return Err(ReaderError::Permission);
}
Ok(())
}
struct SocketPath {
path: PathBuf,
inode: Option<(u64, u64)>,
parent: (u64, u64),
}
impl SocketPath {
fn remove(&mut self) -> Result<(), ReaderError> {
// One cleanup attempt only: never let Drop re-evaluate a later namespace.
let Some(inode) = self.inode.take() else {
return Ok(());
};
let (_, parent) = private_parent(&self.path)?;
if parent != self.parent || fingerprint(&owned_socket(&self.path)?) != inode {
return Err(ReaderError::PathChanged);
}
fs::remove_file(&self.path)?;
Ok(())
}
}
impl Drop for SocketPath {
fn drop(&mut self) {
// Explicit shutdown reports cleanup failure; cancellation preserves detected replacements.
let _ = self.remove();
}
}
/// Signal shutdown, then await the owner's `serve` task to drain workers and clean up.
#[derive(Clone)]
pub struct Shutdown(watch::Sender<bool>);
impl Shutdown {
pub fn shutdown(&self) {
let _ = self.0.send(true);
}
}
pub struct SourceReaderServer {
listener: UnixListener,
writer: NativeLogWriter,
socket: SocketPath,
shutdown: watch::Receiver<bool>,
}
impl SourceReaderServer {
pub fn bind(path: impl AsRef<Path>, writer: NativeLogWriter) -> Result<(Self, Shutdown), ReaderError> {
let (path, parent) = private_parent(path.as_ref())?;
match fs::symlink_metadata(&path) {
Ok(_) => return Err(ReaderError::Occupied),
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(error.into()),
}
// The private parent prevents a foreign UID from observing the initial socket mode.
let listener = UnixListener::bind(&path)?;
let inode = fingerprint(&fs::symlink_metadata(&path)?);
let socket = SocketPath {
path,
inode: Some(inode),
parent,
};
fs::set_permissions(&socket.path, fs::Permissions::from_mode(0o600))?;
if private_parent(&socket.path)?.1 != parent || socket_metadata(&socket.path)? != inode {
return Err(ReaderError::PathChanged);
}
let (sender, shutdown) = watch::channel(false);
Ok((
Self {
listener,
writer,
socket,
shutdown,
},
Shutdown(sender),
))
}
pub async fn serve(mut self) -> Result<(), ReaderError> {
let mut tasks = JoinSet::new();
let outcome = async {
loop {
if *self.shutdown.borrow() {
break;
}
// Admission is shared by every clone of this writer, including re-bound services.
let admission = self.writer.source_reader_admission.clone();
let permit = tokio::select! {
result = admission.acquire_owned() => result.map_err(|_| ReaderError::Protocol)?,
_ = self.shutdown.changed() => break,
};
let stream = tokio::select! {
result = self.listener.accept() => result?.0,
_ = self.shutdown.changed() => break,
};
if peer(&stream).is_err() {
continue;
}
let writer = self.writer.clone();
tasks.spawn(async move {
let _ = connection(stream, writer, permit).await;
});
while let Some(result) = tasks.try_join_next() {
result.map_err(ReaderError::Worker)?;
}
}
Ok(())
}
.await;
// Ordinary shutdown drains connections and blocking workers, never aborts storage work.
while let Some(result) = tasks.join_next().await {
result.map_err(ReaderError::Worker)?;
}
drop(self.listener);
self.socket.remove()?;
outcome
}
}
fn reject(error: Rejection) -> ReadPageReply {
ReadPageReply::Rejected {
version: WIRE_VERSION,
error,
}
}
fn read_page(writer: &NativeLogWriter, request: ReadPage) -> ReadPageReply {
if request.version != WIRE_VERSION
|| request.expected_cluster == [0; 16]
|| request.limit == 0
|| request.limit > crate::MAX_EVENT_PAGE
{
return reject(Rejection::InvalidRequest);
}
// Lock order: connection admission, then Store. Native reads take no writer-ordering lock.
let read = || -> Result<ReadPageReply, NativeWriteError> {
let store = writer.store.lock().map_err(|_| NativeWriteError::Poisoned)?;
let identity = store.durable_source_identity()?;
if identity.cluster_id != request.expected_cluster {
return Ok(reject(Rejection::WrongCluster));
}
let page = match store.created_events_page(request.cursor.as_ref(), request.limit) {
Ok(page) => page,
Err(StoreError::IdentityMismatch) => return Ok(reject(Rejection::InvalidCursor)),
Err(error) => return Err(error.into()),
};
Ok(ReadPageReply::Page {
response: ReadPageResponse {
version: WIRE_VERSION,
identity,
page,
},
})
};
read().unwrap_or_else(|_| reject(Rejection::StoreUnavailable))
}
struct BoundedBytes {
bytes: Vec<u8>,
limit: usize,
}
impl io::Write for BoundedBytes {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
if bytes.len() > self.limit.saturating_sub(self.bytes.len()) {
return Err(io::Error::other("source reader frame limit"));
}
self.bytes.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
fn encode(value: &impl Serialize, limit: usize) -> Result<Vec<u8>, ReaderError> {
let mut buffer = BoundedBytes {
bytes: Vec::new(),
limit,
};
serde_json::to_writer(&mut buffer, value).map_err(|_| ReaderError::Protocol)?;
Ok(buffer.bytes)
}
async fn frame_read(stream: &mut UnixStream, limit: usize) -> Result<Vec<u8>, ReaderError> {
let len = usize::try_from(stream.read_u32().await?).map_err(|_| ReaderError::Protocol)?;
if len == 0 || len > limit {
return Err(ReaderError::Protocol);
}
let mut bytes = vec![0; len];
stream.read_exact(&mut bytes).await?;
Ok(bytes)
}
async fn frame_write(stream: &mut UnixStream, bytes: &[u8]) -> Result<(), ReaderError> {
stream
.write_u32(u32::try_from(bytes.len()).map_err(|_| ReaderError::Protocol)?)
.await?;
stream.write_all(bytes).await?;
Ok(())
}
async fn connection(mut stream: UnixStream, writer: NativeLogWriter, permit: OwnedSemaphorePermit) -> Result<(), ReaderError> {
let bytes = tokio::time::timeout(IO_TIMEOUT, frame_read(&mut stream, MAX_REQUEST_BYTES))
.await
.map_err(|_| ReaderError::Timeout)??;
let request: ReadPage = match serde_json::from_slice(&bytes) {
Ok(request) => request,
Err(_) => {
let bytes = encode(&reject(Rejection::InvalidRequest), MAX_RESPONSE_BYTES)?;
return tokio::time::timeout(IO_TIMEOUT, frame_write(&mut stream, &bytes))
.await
.map_err(|_| ReaderError::Timeout)?;
}
};
// The blocking worker owns admission until read + bounded serialization finish, even
// if a deadline, shutdown or a dropped serve task cancels its awaiting connection.
let (bytes, _permit) = tokio::task::spawn_blocking(move || {
let reply = read_page(&writer, request);
let bytes = encode(&reply, MAX_RESPONSE_BYTES).or_else(|_| encode(&reject(Rejection::ResponseLimit), MAX_RESPONSE_BYTES));
(bytes, permit)
})
.await
.map_err(ReaderError::Worker)?;
tokio::time::timeout(IO_TIMEOUT, frame_write(&mut stream, &bytes?))
.await
.map_err(|_| ReaderError::Timeout)?
}
/// Connect only to a private socket owned by the effective UID, verified at both ends.
#[derive(Clone)]
pub struct SourceReaderClient {
path: PathBuf,
}
impl SourceReaderClient {
pub fn new(path: impl AsRef<Path>) -> Result<Self, ReaderError> {
let (path, _) = private_parent(path.as_ref())?;
socket_metadata(&path)?;
Ok(Self { path })
}
pub async fn page(
&self,
expected_cluster: [u8; 16],
cursor: Option<EventCursor>,
limit: usize,
) -> Result<ReadPageResponse, ReaderError> {
let request = ReadPage {
version: WIRE_VERSION,
expected_cluster,
cursor,
limit,
};
tokio::time::timeout(IO_TIMEOUT, self.page_inner(request))
.await
.map_err(|_| ReaderError::Timeout)?
}
async fn page_inner(&self, request: ReadPage) -> Result<ReadPageResponse, ReaderError> {
let (_, parent) = private_parent(&self.path)?;
let inode = socket_metadata(&self.path)?;
let mut stream = UnixStream::connect(&self.path).await?;
peer(&stream)?;
if private_parent(&self.path)?.1 != parent || socket_metadata(&self.path)? != inode {
return Err(ReaderError::PathChanged);
}
frame_write(&mut stream, &encode(&request, MAX_REQUEST_BYTES)?).await?;
let bytes = frame_read(&mut stream, MAX_RESPONSE_BYTES).await?;
match serde_json::from_slice::<ReadPageReply>(&bytes).map_err(|_| ReaderError::Protocol)? {
ReadPageReply::Page { response } => {
if response.version != WIRE_VERSION
|| response.identity.cluster_id != request.expected_cluster
|| response.identity.cluster_id == [0; 16]
|| response.page.events.len() > request.limit
|| request.limit == 0
|| request.limit > crate::MAX_EVENT_PAGE
{
return Err(ReaderError::Protocol);
}
let next = response
.page
.events
.last()
.map(|event| EventCursor::new(response.identity.cluster_id, event.event_id, &event.captured))
.or(request.cursor);
if response.page.next != next {
return Err(ReaderError::Protocol);
}
Ok(response)
}
ReadPageReply::Rejected { version, error } if version == WIRE_VERSION => Err(ReaderError::Rejected(error)),
_ => Err(ReaderError::Protocol),
}
}
}
#[cfg(test)]
mod cleanup_tests {
use super::*;
#[test]
fn source_reader_cleanup_is_disarmed_before_rebind() {
let directory = tempfile::tempdir().expect("private socket owner");
fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700)).expect("private directory mode");
let (path, parent) = private_parent(&directory.path().join("reader.sock")).expect("private path");
let listener = std::os::unix::net::UnixListener::bind(&path).expect("first real socket");
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).expect("first socket mode");
let inode = socket_metadata(&path).expect("first socket identity");
let mut guard = SocketPath {
path: path.clone(),
inode: Some(inode),
parent,
};
drop(listener);
guard.remove().expect("explicit owner cleanup");
assert!(guard.inode.is_none(), "SOURCE_READER_CLEANUP_DISARMED");
let listener = std::os::unix::net::UnixListener::bind(&path).expect("immediate new owner bind");
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).expect("new socket mode");
let replacement = socket_metadata(&path).expect("new owner socket");
drop(guard);
assert_eq!(
socket_metadata(&path).expect("new socket preserved"),
replacement,
"SOURCE_READER_OLD_GUARD_PRESERVES_REBIND"
);
let _client = std::os::unix::net::UnixStream::connect(&path).expect("new service reachable");
let _connection = listener.accept().expect("new owner accepts");
}
}
@@ -0,0 +1,502 @@
//! Real NativeLogWriter and socket owner is this test process, not a production S3 owner.
use crate::{source_reader::*, *};
use std::{
collections::{BTreeMap, BTreeSet},
os::unix::fs::{MetadataExt, PermissionsExt},
path::PathBuf,
};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::UnixStream,
};
fn identity() -> NodeIdentity {
NodeIdentity {
cluster_id: [7; 16],
node_id: 0,
}
}
fn binding() -> CaptureBinding {
CaptureBinding {
revision: 9,
binding_id: [8; 16],
target_id: [9; 16],
}
}
fn id(index: u64) -> LogId {
LogId {
term: 8,
leader_node: 0,
index,
}
}
fn command(event: u8) -> DecideCreated {
DecideCreated {
operation_id: [event; 16],
object: ObjectIdentity {
bucket_incarnation: [2; 16],
key: vec![event],
},
expected_head: None,
prepared: PreparedIdentity {
version_id: [event; 16],
preparation_id: [4; 16],
content_digest: [5; 32],
content_length: 123,
},
binding_revision: 9,
}
}
fn fixture() -> (tempfile::TempDir, NativeLogWriter) {
let dir = tempfile::tempdir().expect("real source reader directory");
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).expect("private owner directory");
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize real redb");
store.pin_capture_binding(binding()).expect("binding");
store.enable_native_empty().expect("native schema");
store.promote_created_provenance().expect("provenance");
let membership = CaptureMembership::new(
Some(id(0)),
vec![BTreeSet::from([0])],
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("member"))]),
)
.expect("membership");
let membership = openraft::StoredMembership::try_from(&membership).expect("stored membership");
store
.append_native(&[CaptureRaftEntry {
log_id: id(0).into(),
payload: openraft::EntryPayload::Membership(membership.membership().clone()),
}])
.expect("membership log");
store.save_committed(id(0)).expect("membership commit");
store.apply_next_native(id(0)).expect("membership apply");
(dir, NativeLogWriter::new(store))
}
async fn apply(writer: &NativeLogWriter, index: u64, event: u8) {
let entry = CaptureRaftEntry {
log_id: id(index).into(),
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command(event)).expect("canonical native command")),
};
writer
.append_native(vec![entry.clone()])
.await
.expect("live durable log append");
let response = writer.apply_committed(vec![entry]).await.expect("live committed apply");
assert!(
matches!(response[0].created_result(), Some(CreatedResult::Created { .. })),
"SOURCE_READER_REAL_APPLY"
);
}
fn start(
dir: &tempfile::TempDir,
writer: &NativeLogWriter,
) -> (PathBuf, SourceReaderClient, Shutdown, tokio::task::JoinHandle<Result<(), ReaderError>>) {
let path = dir.path().join("reader.sock");
let (server, shutdown) = SourceReaderServer::bind(&path, writer.clone()).expect("owner socket bind");
let client = SourceReaderClient::new(&path).expect("trusted local client");
let task = tokio::spawn(server.serve());
(path, client, shutdown, task)
}
async fn stop(path: PathBuf, shutdown: Shutdown, task: tokio::task::JoinHandle<Result<(), ReaderError>>) {
shutdown.shutdown();
task.await.expect("serve task").expect("clean shutdown");
assert!(!path.exists(), "SOURCE_READER_SOCKET_CLEANUP");
}
async fn raw(path: &std::path::Path, value: serde_json::Value) -> ReadPageReply {
let mut stream = UnixStream::connect(path).await.expect("real socket connect");
let bytes = serde_json::to_vec(&value).expect("wire request");
stream
.write_u32(bytes.len().try_into().expect("request length"))
.await
.expect("frame prefix");
stream.write_all(&bytes).await.expect("frame bytes");
let len = stream.read_u32().await.expect("reply prefix");
let mut bytes = vec![0; usize::try_from(len).expect("reply length")];
stream.read_exact(&mut bytes).await.expect("reply bytes");
serde_json::from_slice(&bytes).expect("strict native reply")
}
fn request() -> serde_json::Value {
serde_json::to_value(ReadPage {
version: WIRE_VERSION,
expected_cluster: identity().cluster_id,
cursor: None,
limit: 2,
})
.expect("native request")
}
#[tokio::test]
async fn source_reader_live_apply() {
let (dir, writer) = fixture();
let (path, client, shutdown, task) = start(&dir, &writer);
assert_eq!(writer.durable_identity().await.expect("durable META identity"), identity());
assert!(
client
.page(identity().cluster_id, None, 2)
.await
.expect("empty live page")
.page
.events
.is_empty()
);
apply(&writer, 1, 9).await;
let response = client
.page(identity().cluster_id, None, 2)
.await
.expect("live committed page");
assert_eq!(response.identity, identity(), "SOURCE_READER_DURABLE_IDENTITY");
assert_eq!(response.page.events[0].event_id, [9; 16], "SOURCE_READER_LIVE_EVENT");
assert_eq!(response.page.events[0].captured.binding, binding());
{
let mut corrupt = writer.store.lock().expect("test owner");
let tx = corrupt.write_transaction().expect("corrupt META transaction");
tx.open_table(META)
.expect("META")
.insert("identity", &[0; 24][..])
.expect("corrupt identity");
corrupt.commit(tx).expect("persist corruption");
}
assert!(writer.durable_identity().await.is_err(), "SOURCE_READER_META_CORRUPTION_REJECTED");
assert!(
matches!(
client.page(identity().cluster_id, None, 2).await,
Err(ReaderError::Rejected(Rejection::StoreUnavailable))
),
"SOURCE_READER_NO_CALLER_IDENTITY_SUBSTITUTION"
);
stop(path, shutdown, task).await;
}
#[tokio::test]
async fn source_reader_restart_cursor() {
let (dir, writer) = fixture();
apply(&writer, 1, 9).await;
apply(&writer, 2, 7).await;
let (path, client, shutdown, task) = start(&dir, &writer);
let first = client.page(identity().cluster_id, None, 1).await.expect("first");
stop(path, shutdown, task).await;
drop(writer);
let writer = NativeLogWriter::new(Store::open(dir.path().join("db"), identity()).expect("restart same durable store"));
let (path, client, shutdown, task) = start(&dir, &writer);
let next = client
.page(identity().cluster_id, first.page.next, 1)
.await
.expect("resume durable cursor");
assert_eq!(next.page.events[0].event_id, [7; 16], "SOURCE_READER_RESTART_CONTINUES");
stop(path, shutdown, task).await;
}
#[tokio::test]
async fn source_reader_page_boundaries() {
let (dir, writer) = fixture();
for (index, event) in [(1, 9), (2, 7), (3, 1)] {
apply(&writer, index, event).await;
}
let (path, client, shutdown, task) = start(&dir, &writer);
let first = client.page(identity().cluster_id, None, 2).await.expect("first two");
assert_eq!(
first.page.events.iter().map(|e| e.event_id).collect::<Vec<_>>(),
vec![[9; 16], [7; 16]],
"SOURCE_READER_DECISION_ORDER"
);
let next = client
.page(identity().cluster_id, first.page.next, 2)
.await
.expect("remaining");
assert_eq!(next.page.events.len(), 1);
let end = client
.page(identity().cluster_id, next.page.next.clone(), 2)
.await
.expect("end");
assert!(end.page.events.is_empty());
assert_eq!(end.page.next, next.page.next, "SOURCE_READER_CURSOR_RETAINED_AT_END");
for limit in [0, MAX_EVENT_PAGE + 1] {
assert!(
matches!(
client.page(identity().cluster_id, None, limit).await,
Err(ReaderError::Rejected(Rejection::InvalidRequest))
),
"SOURCE_READER_LIMIT_REJECTED"
);
}
stop(path, shutdown, task).await;
}
#[tokio::test]
async fn source_reader_wrong_identity() {
let (dir, writer) = fixture();
apply(&writer, 1, 9).await;
let (path, client, shutdown, task) = start(&dir, &writer);
let mut wrong = request();
wrong["expected_cluster"] = serde_json::to_value([6; 16]).expect("wrong cluster");
assert!(
matches!(
raw(&path, wrong).await,
ReadPageReply::Rejected {
error: Rejection::WrongCluster,
..
}
),
"SOURCE_READER_CLUSTER: reject before reading or returning foreign events"
);
let first = client.page(identity().cluster_id, None, 1).await.expect("real cursor");
let mut wrong = serde_json::to_value(first.page.next.expect("cursor")).expect("cursor wire");
wrong["event"] = serde_json::to_value([1; 16]).expect("wrong event");
let wrong: EventCursor = serde_json::from_value(wrong).expect("syntactically valid foreign cursor");
assert!(
matches!(
client.page(identity().cluster_id, Some(wrong), 1).await,
Err(ReaderError::Rejected(Rejection::InvalidCursor))
),
"SOURCE_READER_CURSOR_BOUND"
);
for (field, value) in [
("version", serde_json::json!(2)),
("unknown", serde_json::json!(true)),
("cursor", serde_json::json!({"unknown":true})),
] {
let mut wrong = request();
wrong[field] = value;
assert!(
matches!(
raw(&path, wrong).await,
ReadPageReply::Rejected {
error: Rejection::InvalidRequest,
..
}
),
"SOURCE_READER_STRICT_REQUEST"
);
}
stop(path, shutdown, task).await;
}
#[tokio::test]
async fn source_reader_socket_ownership() {
let (dir, writer) = fixture();
let path = dir.path().join("reader.sock");
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o750)).expect("insecure directory");
assert!(
matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Permission)),
"SOURCE_READER_PRIVATE_DIRECTORY"
);
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).expect("restore private directory");
std::fs::write(&path, b"preserve me").expect("occupied file");
assert!(
matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Occupied)),
"SOURCE_READER_OCCUPIED_PATH"
);
assert_eq!(std::fs::read(&path).expect("unchanged file"), b"preserve me");
std::fs::remove_file(&path).expect("remove own fixture");
std::os::unix::fs::symlink(dir.path().join("missing"), &path).expect("unknown symlink");
assert!(matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Occupied)));
assert!(SourceReaderClient::new(&path).is_err());
std::fs::remove_file(&path).expect("remove own symlink");
let long = dir.path().join("x".repeat(300));
assert!(SourceReaderServer::bind(&long, writer.clone()).is_err(), "SOURCE_READER_LONG_SOCKET_PATH");
let parent_link = dir.path().join("parent-link");
std::os::unix::fs::symlink(dir.path(), &parent_link).expect("parent symlink");
assert!(
matches!(
SourceReaderServer::bind(parent_link.join("socket"), writer.clone()),
Err(ReaderError::Permission)
),
"SOURCE_READER_PARENT_SYMLINK_DENIED"
);
let (path, client, shutdown, task) = start(&dir, &writer);
let meta = std::fs::symlink_metadata(&path).expect("socket metadata");
assert_eq!(meta.mode() & 0o777, 0o600);
assert_eq!(meta.uid(), rustix::process::geteuid().as_raw());
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o660)).expect("insecure socket");
assert!(
matches!(client.page(identity().cluster_id, None, 1).await, Err(ReaderError::Permission)),
"SOURCE_READER_CLIENT_VALIDATES_SOCKET"
);
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("restore socket");
std::fs::remove_file(&path).expect("test owner replaces socket");
std::fs::write(&path, b"replacement").expect("replacement file");
shutdown.shutdown();
assert!(task.await.expect("serve exit").is_err(), "SOURCE_READER_REPLACEMENT_CLEANUP_REJECTED");
assert_eq!(std::fs::read(&path).expect("replacement preserved"), b"replacement");
}
#[tokio::test]
async fn source_reader_transport_bounds() {
let (dir, writer) = fixture();
let (path, client, shutdown, task) = start(&dir, &writer);
let mut stream = UnixStream::connect(&path).await.expect("oversized input peer");
stream
.write_u32(u32::try_from(MAX_REQUEST_BYTES + 1).expect("length"))
.await
.expect("overlong prefix");
let mut byte = [0];
assert_eq!(
stream.read(&mut byte).await.expect("bounded rejection closes"),
0,
"SOURCE_READER_INPUT_BOUND"
);
let mut idle = UnixStream::connect(&path).await.expect("idle connection");
let start = std::time::Instant::now();
assert_eq!(
idle.read(&mut byte).await.expect("deadline closes connection"),
0,
"SOURCE_READER_READ_DEADLINE"
);
assert!(start.elapsed() >= IO_TIMEOUT, "SOURCE_READER_REAL_READ_DEADLINE");
assert!(
client.page(identity().cluster_id, None, 1).await.is_ok(),
"SOURCE_READER_TIMEOUT_RELEASES_CONNECTION"
);
stop(path.clone(), shutdown, task).await;
// A same-UID peer still cannot supply unbounded responses to the client.
let listener = tokio::net::UnixListener::bind(&path).expect("controlled response peer");
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("private response socket");
let client = SourceReaderClient::new(&path).expect("response client");
let peer = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("client connects");
stream
.write_u32(u32::try_from(MAX_RESPONSE_BYTES + 1).expect("oversized response"))
.await
.expect("response prefix");
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
});
assert!(
matches!(client.page(identity().cluster_id, None, 1).await, Err(ReaderError::Protocol)),
"SOURCE_READER_RESPONSE_BOUND"
);
peer.await.expect("peer task");
}
fn hold_store(writer: &NativeLogWriter) -> (std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>) {
let owner = writer.clone();
let (locked_tx, locked_rx) = std::sync::mpsc::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel();
let thread = std::thread::spawn(move || {
let _store = owner.store.lock().expect("actual Store held by test owner");
locked_tx.send(()).expect("lock visible");
let _ = release_rx.recv();
});
locked_rx.recv().expect("store locked");
(release_tx, thread)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn source_reader_shutdown() {
let (dir, writer) = fixture();
apply(&writer, 1, 9).await;
let (path, _client, shutdown, task) = start(&dir, &writer);
let (release_tx, blocker) = hold_store(&writer);
let mut streams = Vec::new();
for _ in 0..MAX_CONNECTIONS {
let mut stream = UnixStream::connect(&path).await.expect("active read socket");
let bytes = serde_json::to_vec(&request()).expect("request");
stream
.write_u32(bytes.len().try_into().expect("length"))
.await
.expect("prefix");
stream.write_all(&bytes).await.expect("request bytes");
streams.push(stream);
}
tokio::time::timeout(std::time::Duration::from_secs(2), async {
while writer.source_reader_admission.available_permits() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("all admissions active");
drop(streams);
tokio::time::sleep(IO_TIMEOUT + std::time::Duration::from_millis(100)).await;
assert_eq!(
writer.source_reader_admission.available_permits(),
0,
"SOURCE_READER_CLIENT_TIMEOUT_KEEPS_WORKER_ADMISSION"
);
shutdown.shutdown();
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert!(!task.is_finished(), "SOURCE_READER_SHUTDOWN_DRAINS_READERS");
assert_eq!(
writer.source_reader_admission.available_permits(),
0,
"SOURCE_READER_CANCELLED_CLIENTS_KEEP_ADMISSION"
);
release_tx.send(()).expect("release real blocking reads");
blocker.join().expect("test lock owner exits");
task.await.expect("serve completes").expect("shutdown clean");
assert!(!path.exists());
assert_eq!(
writer.source_reader_admission.available_permits(),
MAX_CONNECTIONS,
"SOURCE_READER_ADMISSION_RELEASED_AFTER_WORKERS"
);
let (release, blocker) = hold_store(&writer);
let (path, _, _shutdown, task) = start(&dir, &writer);
let bytes = serde_json::to_vec(&request()).expect("native request");
let mut sockets = Vec::new();
for _ in 0..MAX_CONNECTIONS {
let mut stream = UnixStream::connect(&path).await.expect("cancelled service peer");
stream
.write_u32(bytes.len().try_into().expect("length"))
.await
.expect("prefix");
stream.write_all(&bytes).await.expect("request");
sockets.push(stream);
}
tokio::time::timeout(std::time::Duration::from_secs(2), async {
while writer.source_reader_admission.available_permits() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled service admits bounded workers");
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
task.abort();
assert!(task.await.expect_err("serve cancellation").is_cancelled());
drop(sockets);
assert_eq!(
writer.source_reader_admission.available_permits(),
0,
"SOURCE_READER_CANCELLED_SERVE_RETAINS_WORKER_ADMISSION"
);
let (path, client, shutdown, task) = start(&dir, &writer);
let pending = tokio::spawn(async move { client.page(identity().cluster_id, None, 1).await });
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert!(!pending.is_finished(), "SOURCE_READER_REBIND_CANNOT_BYPASS_ADMISSION");
assert_eq!(writer.source_reader_admission.available_permits(), 0);
release.send(()).expect("release cancelled worker store lock");
blocker.join().expect("lock owner completes");
assert_eq!(
pending
.await
.expect("new service client")
.expect("new service waits for old workers")
.page
.events
.len(),
1
);
stop(path, shutdown, task).await;
}
#[tokio::test]
async fn source_reader_checkpoint_replay() {
let (dir, writer) = fixture();
apply(&writer, 1, 9).await;
apply(&writer, 2, 7).await;
let (path, client, shutdown, task) = start(&dir, &writer);
let first = client
.page(identity().cluster_id, None, 1)
.await
.expect("pre-checkpoint cursor");
let checkpoint = writer
.checkpoint_local(CheckpointLimits::default())
.await
.expect("real checkpoint");
assert_eq!(checkpoint.cut(), id(2));
let next = client
.page(identity().cluster_id, first.page.next.clone(), 1)
.await
.expect("checkpoint retained events");
assert_eq!(next.page.events[0].event_id, [7; 16], "SOURCE_READER_CHECKPOINT_REPLAY");
stop(path, shutdown, task).await;
drop(writer);
let writer = NativeLogWriter::new(Store::open(dir.path().join("db"), identity()).expect("reopen checkpointed store"));
let (path, client, shutdown, task) = start(&dir, &writer);
assert_eq!(
client
.page(identity().cluster_id, first.page.next, 1)
.await
.expect("replay after restart"),
next,
"SOURCE_READER_CHECKPOINT_RESTART"
);
stop(path, shutdown, task).await;
}
+6 -2
View File
@@ -21,8 +21,12 @@ impl Store {
None
} else {
let index = from - 1;
let record = logs.get(index)?.ok_or(StoreError::InvalidLogSequence)?;
let id = decode_log(record.value())?.id;
let id = match logs.get(index)? {
Some(record) => decode_log(record.value())?.id,
None => crate::checkpoint_baseline::read_purged(&meta)?
.filter(|id| id.index == index)
.ok_or(StoreError::InvalidLogSequence)?,
};
if id.index != index {
return Err(StoreError::CorruptRecord);
}
+2
View File
@@ -122,6 +122,8 @@ test-util = []
e2e-test-hooks = []
[dependencies]
rustfs-capture-authority = { workspace = true }
openraft = { workspace = true, features = ["storage-v2"] }
hotpath.workspace = true
rustfs-filemeta.workspace = true
rustfs-utils = { workspace = true, features = ["full"] }
+26
View File
@@ -372,6 +372,25 @@ pub mod config {
with_server_config_write_lock,
};
/// Observe persisted config bytes and their reader-associated metadata within a byte limit.
pub async fn read_config_observation(
api: std::sync::Arc<crate::store::ECStore>,
file: &str,
max_bytes: usize,
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
crate::config::com::read_config_observation(api, file, max_bytes).await
}
/// Observe bounded config bytes while the caller retains the object's write owner.
pub async fn read_config_observation_no_lock(
api: std::sync::Arc<crate::store::ECStore>,
file: &str,
max_bytes: usize,
guard: &crate::object_api::WriteCommitGuard,
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
crate::config::com::read_config_observation_no_lock(api, file, max_bytes, guard).await
}
/// Read config bytes and metadata without treating an empty payload as absent.
pub async fn read_config_preserve_empty_with_metadata_opts(
api: std::sync::Arc<crate::store::ECStore>,
@@ -578,6 +597,7 @@ pub mod rio {
}
pub mod rpc {
pub use crate::cluster::rpc::capture_raft::{CaptureRaftError, CaptureRaftLimits, CaptureRaftNetworkFactory, CaptureRaftRpc};
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
@@ -624,7 +644,13 @@ pub mod store_list {
pub mod storage {
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
pub use crate::disk::local::native_prepare::{
NativePreparationDescription, NativePreparationProbe, NativePreparationReceipt,
};
pub use crate::store::HealWalkVersion;
pub use crate::store::native_prepare::{
NativePreparationFinalizeRequest, NativePreparationRecovery, NativePreparationTarget,
};
pub use crate::store::{
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
@@ -0,0 +1,806 @@
//! Bounded authenticated Raft RPC, mountable without a second storage owner.
use super::{client, http_auth};
use openraft::{
DefensiveError, ErrorSubject, ErrorVerb, RaftNetwork, RaftNetworkFactory, StorageError, StorageIOError, Violation,
error::{Fatal, InstallSnapshotError, RPCError, RaftError, RemoteError, Unreachable},
network::RPCOption,
raft::{
AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, VoteRequest, VoteResponse,
},
};
use rustfs_capture_authority::{CaptureMembership, CaptureNode, CaptureTypeConfig, NativeRaftOwner, NodeIdentity};
use rustfs_protos::proto_gen::node_service::{
CaptureRaftRequest, CaptureRaftResponse,
capture_raft_service_server::{CaptureRaftService, CaptureRaftServiceServer},
};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::value::RawValue;
use std::{collections::BTreeMap, io::Write, sync::Arc, time::Duration};
use tonic::{Request, Response, Status};
#[derive(Clone, Copy, Debug)]
pub struct CaptureRaftLimits {
pub max_request_bytes: usize,
pub max_response_bytes: usize,
pub max_entries: usize,
pub max_snapshot_chunk_bytes: usize,
pub max_inflight: usize,
pub connect_timeout: Duration,
pub request_timeout: Duration,
}
impl Default for CaptureRaftLimits {
fn default() -> Self {
Self {
max_request_bytes: 16 * 1024 * 1024,
max_response_bytes: 1024 * 1024,
max_entries: 300,
max_snapshot_chunk_bytes: 3 * 1024 * 1024,
max_inflight: 16,
connect_timeout: Duration::from_secs(3),
request_timeout: Duration::from_secs(5),
}
}
}
impl CaptureRaftLimits {
fn validate(&self) -> Result<(), CaptureRaftError> {
let chunk = self
.max_snapshot_chunk_bytes
.checked_mul(4)
.and_then(|n| n.checked_add(16384))
.ok_or(CaptureRaftError::Limits)?;
if self.max_request_bytes < chunk
|| self.max_request_bytes > 64 * 1024 * 1024
|| self.max_response_bytes < 4096
|| self.max_response_bytes > 4 * 1024 * 1024
|| self.max_entries == 0
|| self.max_entries > 1024
|| self.max_snapshot_chunk_bytes == 0
|| self.max_inflight == 0
|| self.max_inflight > 128
|| self.connect_timeout.is_zero()
|| self.request_timeout.is_zero()
|| self.request_timeout > Duration::from_secs(60)
|| self.connect_timeout > Duration::from_secs(60)
{
return Err(CaptureRaftError::Limits);
}
Ok(())
}
}
#[derive(Debug, thiserror::Error)]
pub enum CaptureRaftError {
#[error("capture Raft message limit exceeded")]
Limits,
#[error("capture Raft request requires a smaller append batch")]
RequestTooLarge,
#[error("invalid capture Raft message")]
Protocol,
#[error("capture Raft identity mismatch")]
Identity,
#[error("capture Raft authentication failed")]
Authentication,
#[error("capture Raft channel unavailable")]
ChannelUnavailable,
#[error("capture Raft deadline exceeded")]
Deadline,
#[error(transparent)]
Io(#[from] std::io::Error),
#[error(transparent)]
Json(#[from] serde_json::Error),
#[error(transparent)]
Node(#[from] rustfs_capture_authority::CaptureNodeError),
#[error(transparent)]
Status(Box<Status>),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) enum Method {
AppendEntries,
Vote,
InstallSnapshot,
}
impl Method {
pub(crate) fn path(self) -> &'static str {
match self {
Self::AppendEntries => "/node_service.CaptureRaftService/AppendEntries",
Self::Vote => "/node_service.CaptureRaftService/Vote",
Self::InstallSnapshot => "/node_service.CaptureRaftService/InstallSnapshot",
}
}
}
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct WireRequest {
pub version: u32,
pub cluster: [u8; 16],
pub source: u64,
pub target: u64,
pub method: Method,
pub nonce: [u8; 16],
pub payload: Box<RawValue>,
}
struct Bounded {
bytes: Vec<u8>,
max: usize,
}
impl Write for Bounded {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
let end = self
.bytes
.len()
.checked_add(bytes.len())
.ok_or_else(|| std::io::Error::other("capture RPC limit"))?;
if end > self.max {
return Err(std::io::Error::other("capture RPC limit"));
}
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
self.bytes.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
fn encoded<T: Serialize>(value: &T, max: usize) -> Result<Vec<u8>, CaptureRaftError> {
let mut out = Bounded { bytes: Vec::new(), max };
serde_json::to_writer(&mut out, value).map_err(|error| {
if error.is_io() {
CaptureRaftError::Limits
} else {
CaptureRaftError::Json(error)
}
})?;
Ok(out.bytes)
}
fn raw<T: Serialize>(value: &T, max: usize) -> Result<Box<RawValue>, CaptureRaftError> {
Ok(RawValue::from_string(
String::from_utf8(encoded(value, max)?).map_err(|_| CaptureRaftError::Protocol)?,
)?)
}
pub(crate) fn encode_request<P: Serialize>(
local: NodeIdentity,
target: u64,
method: Method,
payload: &P,
limits: CaptureRaftLimits,
) -> Result<CaptureRaftRequest, CaptureRaftError> {
let wire = WireRequest {
version: 1,
cluster: local.cluster_id,
source: local.node_id,
target,
method,
nonce: *uuid::Uuid::new_v4().as_bytes(),
payload: raw(payload, limits.max_request_bytes)?,
};
Ok(CaptureRaftRequest {
body: encoded(&wire, limits.max_request_bytes)?.into(),
})
}
pub(crate) fn canonical_request(body: &[u8]) -> Vec<u8> {
let mut out = b"rustfs.capture-raft.request.v1\0".to_vec();
out.extend_from_slice(body);
out
}
fn decode_wire(body: &[u8], max: usize) -> Result<WireRequest, CaptureRaftError> {
if body.len() > max {
return Err(CaptureRaftError::Limits);
}
let wire: WireRequest = serde_json::from_slice(body)?;
if wire.version != 1 || wire.cluster == [0; 16] || wire.nonce == [0; 16] || encoded(&wire, max)? != body {
return Err(CaptureRaftError::Protocol);
}
Ok(wire)
}
fn decode_payload<P: Serialize + DeserializeOwned>(wire: &WireRequest, max: usize) -> Result<P, CaptureRaftError> {
let value: P = serde_json::from_str(wire.payload.get())?;
if encoded(&value, max)?.as_slice() != wire.payload.get().as_bytes() {
return Err(CaptureRaftError::Protocol);
}
Ok(value)
}
#[derive(Serialize, Deserialize)]
#[serde(tag = "kind", deny_unknown_fields)]
pub(crate) enum RemoteFailure {
Stopped,
Panicked,
StorageIo {
operation: Option<IoOperation>,
},
StorageDefensive {
subject: ErrorSubject<u64>,
violation: Violation<u64>,
},
SnapshotMismatch {
error: InstallSnapshotError,
},
}
#[derive(Serialize, Deserialize)]
pub(crate) struct IoOperation {
subject: ErrorSubject<u64>,
verb: ErrorVerb,
}
fn fatal(error: Fatal<u64>) -> RemoteFailure {
match error {
Fatal::Stopped => RemoteFailure::Stopped,
Fatal::Panicked => RemoteFailure::Panicked,
Fatal::StorageError(StorageError::IO { source }) => RemoteFailure::StorageIo {
operation: encoded(&source, 16384)
.ok()
.and_then(|bytes| serde_json::from_slice(&bytes).ok()),
},
Fatal::StorageError(StorageError::Defensive { source }) => RemoteFailure::StorageDefensive {
subject: source.subject,
violation: source.violation,
},
}
}
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
enum WireReply {
Success { payload: Box<RawValue> },
Failure { error: RemoteFailure },
}
fn response_context(request: &CaptureRaftRequest, body: &[u8]) -> Result<Vec<u8>, CaptureRaftError> {
let mut out = b"rustfs.capture-raft.response.v1\0".to_vec();
out.try_reserve(
request
.body
.len()
.checked_add(body.len())
.and_then(|n| n.checked_add(16))
.ok_or(CaptureRaftError::Limits)?,
)
.map_err(std::io::Error::other)?;
out.extend(
u64::try_from(request.body.len())
.map_err(|_| CaptureRaftError::Limits)?
.to_be_bytes(),
);
out.extend_from_slice(&request.body);
out.extend(u64::try_from(body.len()).map_err(|_| CaptureRaftError::Limits)?.to_be_bytes());
out.extend_from_slice(body);
Ok(out)
}
fn reply<P: Serialize>(
request: &CaptureRaftRequest,
result: Result<P, RemoteFailure>,
limits: CaptureRaftLimits,
) -> Result<CaptureRaftResponse, CaptureRaftError> {
let wire = match result {
Ok(value) => WireReply::Success {
payload: raw(&value, limits.max_response_bytes)?,
},
Err(error) => WireReply::Failure { error },
};
let body = encoded(&wire, limits.max_response_bytes)?;
let response_proof = http_auth::sign_tonic_rpc_response_proof(&response_context(request, &body)?)?;
Ok(CaptureRaftResponse {
body: body.into(),
response_proof: response_proof.into(),
})
}
pub(crate) fn verify_response<P: Serialize + DeserializeOwned>(
request: &CaptureRaftRequest,
response: CaptureRaftResponse,
limits: CaptureRaftLimits,
) -> Result<Result<P, RemoteFailure>, CaptureRaftError> {
if request.body.len() > limits.max_request_bytes
|| response.body.len() > limits.max_response_bytes
|| response.response_proof.len() != 32
{
return Err(CaptureRaftError::Limits);
}
http_auth::verify_tonic_rpc_response_proof(&response_context(request, &response.body)?, &response.response_proof)?;
let wire: WireReply = serde_json::from_slice(&response.body)?;
if encoded(&wire, limits.max_response_bytes)?.as_slice() != response.body {
return Err(CaptureRaftError::Protocol);
}
match wire {
WireReply::Failure { error } => Ok(Err(error)),
WireReply::Success { payload } => {
let value: P = serde_json::from_str(payload.get())?;
if encoded(&value, limits.max_response_bytes)?.as_slice() != payload.get().as_bytes() {
return Err(CaptureRaftError::Protocol);
}
Ok(Ok(value))
}
}
}
fn restored_fatal(error: RemoteFailure) -> Result<Fatal<u64>, CaptureRaftError> {
Ok(match error {
RemoteFailure::Stopped => Fatal::Stopped,
RemoteFailure::Panicked => Fatal::Panicked,
RemoteFailure::StorageIo { operation } => {
let (subject, verb) = operation
.map(|op| (op.subject, op.verb))
.unwrap_or((ErrorSubject::None, ErrorVerb::Read));
Fatal::StorageError(
StorageIOError::new(
subject,
verb,
openraft::AnyError::new(&std::io::Error::other("remote Raft storage I/O failed; detailed source redacted")),
)
.into(),
)
}
RemoteFailure::StorageDefensive { subject, violation } => Fatal::StorageError(StorageError::Defensive {
source: DefensiveError {
subject,
violation,
backtrace: None,
},
}),
RemoteFailure::SnapshotMismatch { .. } => return Err(CaptureRaftError::Protocol),
})
}
fn peers_valid(local: NodeIdentity, peers: &BTreeMap<u64, CaptureNode>) -> Result<(), CaptureRaftError> {
if local.cluster_id == [0; 16] || !peers.contains_key(&local.node_id) {
return Err(CaptureRaftError::Identity);
}
let mut addresses = std::collections::BTreeSet::new();
for node in peers.values() {
node.validate()?;
if !addresses.insert(node.rpc_uri()) {
return Err(CaptureRaftError::Identity);
}
}
Ok(())
}
#[cfg(test)]
type WorkerPause = Arc<std::sync::Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>>;
#[derive(Clone)]
pub struct CaptureRaftRpc {
owner: Arc<NativeRaftOwner>,
peers: Arc<BTreeMap<u64, CaptureNode>>,
audience: String,
limits: CaptureRaftLimits,
admission: Arc<tokio::sync::Semaphore>,
accepting: Arc<std::sync::atomic::AtomicBool>,
#[cfg(test)]
pub(crate) worker_pause: WorkerPause,
}
impl CaptureRaftRpc {
pub fn new(
owner: Arc<NativeRaftOwner>,
peers: BTreeMap<u64, CaptureNode>,
limits: CaptureRaftLimits,
) -> Result<Self, CaptureRaftError> {
limits.validate()?;
let identity = owner.identity();
peers_valid(identity, &peers)?;
let node = peers.get(&identity.node_id).ok_or(CaptureRaftError::Identity)?;
let uri: http::Uri = node.rpc_uri().parse().map_err(|_| CaptureRaftError::Identity)?;
let audience = http_auth::normalize_tonic_rpc_audience(uri.authority().ok_or(CaptureRaftError::Identity)?.as_str())?;
Ok(Self {
owner,
peers: Arc::new(peers),
audience,
limits,
#[cfg(test)]
worker_pause: Arc::new(std::sync::Mutex::new(None)),
accepting: Arc::new(std::sync::atomic::AtomicBool::new(true)),
admission: Arc::new(tokio::sync::Semaphore::new(limits.max_inflight)),
})
}
pub fn into_service(self) -> CaptureRaftServiceServer<Self> {
let request = self.limits.max_request_bytes + 64;
let response = self.limits.max_response_bytes + 128;
CaptureRaftServiceServer::new(self)
.max_decoding_message_size(request)
.max_encoding_message_size(response)
}
fn prepare<P: Serialize + DeserializeOwned>(
&self,
request: &Request<CaptureRaftRequest>,
method: Method,
) -> Result<(WireRequest, P, tokio::sync::OwnedSemaphorePermit), Status> {
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
return Err(Status::unavailable("capture Raft service stopping"));
}
let permit = self
.admission
.clone()
.try_acquire_owned()
.map_err(|_| Status::resource_exhausted("capture Raft admission full"))?;
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
return Err(Status::unavailable("capture Raft service stopping"));
}
if request.get_ref().body.len() > self.limits.max_request_bytes {
return Err(Status::resource_exhausted("capture Raft request limit"));
}
if request
.metadata()
.get(http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER)
.and_then(|v| v.to_str().ok())
!= Some("3")
{
return Err(auth_rejected("missing_replay_scope"));
}
http_auth::verify_tonic_rpc_signature_with_bootstrap(&self.audience, method.path(), request.metadata().as_ref(), false)
.map_err(|_| auth_rejected("invalid_signature"))?;
http_auth::verify_tonic_canonical_body_digest(request, &canonical_request(&request.get_ref().body))
.map_err(|_| auth_rejected("body_digest_mismatch"))?;
let wire = decode_wire(&request.get_ref().body, self.limits.max_request_bytes)
.map_err(|_| Status::invalid_argument("capture Raft wire rejected"))?;
let local = self.owner.identity();
if wire.cluster != local.cluster_id
|| wire.target != local.node_id
|| wire.method != method
|| !self.peers.contains_key(&wire.source)
{
return Err(Status::permission_denied("capture Raft identity rejected"));
}
let payload = decode_payload(&wire, self.limits.max_request_bytes)
.map_err(|_| Status::invalid_argument("capture Raft payload rejected"))?;
Ok((wire, payload, permit))
}
#[cfg(test)]
async fn paused(pause: WorkerPause) {
let gate = pause.lock().expect("worker pause lock").take();
if let Some((entered, resume)) = gate {
entered.notify_one();
resume.notified().await;
}
}
/// Stop admission and wait for all actual RPC tasks, including cancelled HTTP waiters.
pub async fn shutdown(&self) -> Result<(), CaptureRaftError> {
self.accepting.store(false, std::sync::atomic::Ordering::Release);
let count = u32::try_from(self.limits.max_inflight).map_err(|_| CaptureRaftError::Limits)?;
let _all = self
.admission
.clone()
.acquire_many_owned(count)
.await
.map_err(|_| CaptureRaftError::ChannelUnavailable)?;
Ok(())
}
}
fn signed_response<P: Serialize>(
raw: &CaptureRaftRequest,
result: Result<P, RemoteFailure>,
limits: CaptureRaftLimits,
audience: &str,
challenge: Option<uuid::Uuid>,
) -> Result<Response<CaptureRaftResponse>, Status> {
let mut response =
Response::new(reply(raw, result, limits).map_err(|_| Status::internal("capture Raft response unavailable"))?);
if let Some(challenge) = challenge {
response.metadata_mut().as_mut().extend(
http_auth::tonic_boot_epoch_response_headers(audience, challenge)
.map_err(|_| Status::internal("capture Raft epoch proof unavailable"))?,
);
}
Ok(response)
}
#[tonic::async_trait]
impl CaptureRaftService for CaptureRaftRpc {
async fn append_entries(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
let (wire, payload, permit) = self.prepare::<AppendEntriesRequest<CaptureTypeConfig>>(&request, Method::AppendEntries)?;
validate_append(&payload, wire.source, self.limits)
.map_err(|_| Status::invalid_argument("capture Raft append rejected"))?;
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
let raw = request.into_inner();
let owner = self.owner.clone();
let audience = self.audience.clone();
let limits = self.limits;
#[cfg(test)]
let pause = self.worker_pause.clone();
tokio::spawn(async move {
let _permit = permit;
#[cfg(test)]
Self::paused(pause).await;
let result = owner.raft().append_entries(payload).await.map_err(|error| match error {
RaftError::Fatal(error) => fatal(error),
RaftError::APIError(error) => match error {},
});
drop(owner);
signed_response(&raw, result, limits, &audience, challenge)
})
.await
.map_err(|_| Status::internal("capture Raft worker failed"))?
}
async fn vote(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
let (wire, payload, permit) = self.prepare::<VoteRequest<u64>>(&request, Method::Vote)?;
if payload.vote.leader_id.node_id != wire.source {
return Err(Status::invalid_argument("capture Raft candidate rejected"));
}
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
let raw = request.into_inner();
let owner = self.owner.clone();
let audience = self.audience.clone();
let limits = self.limits;
#[cfg(test)]
let pause = self.worker_pause.clone();
tokio::spawn(async move {
let _permit = permit;
#[cfg(test)]
Self::paused(pause).await;
let result = owner.raft().vote(payload).await.map_err(|error| match error {
RaftError::Fatal(error) => fatal(error),
RaftError::APIError(error) => match error {},
});
drop(owner);
signed_response(&raw, result, limits, &audience, challenge)
})
.await
.map_err(|_| Status::internal("capture Raft worker failed"))?
}
async fn install_snapshot(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
let (wire, payload, permit) =
self.prepare::<InstallSnapshotRequest<CaptureTypeConfig>>(&request, Method::InstallSnapshot)?;
validate_snapshot(&payload, wire.source, self.limits)
.map_err(|_| Status::invalid_argument("capture Raft snapshot rejected"))?;
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
let raw = request.into_inner();
let owner = self.owner.clone();
let audience = self.audience.clone();
let limits = self.limits;
#[cfg(test)]
let pause = self.worker_pause.clone();
tokio::spawn(async move {
let _permit = permit;
#[cfg(test)]
Self::paused(pause).await;
let result = owner.raft().install_snapshot(payload).await.map_err(|error| match error {
RaftError::Fatal(error) => fatal(error),
RaftError::APIError(error) => RemoteFailure::SnapshotMismatch { error },
});
drop(owner);
signed_response(&raw, result, limits, &audience, challenge)
})
.await
.map_err(|_| Status::internal("capture Raft worker failed"))?
}
}
fn validate_append(
request: &AppendEntriesRequest<CaptureTypeConfig>,
source: u64,
limits: CaptureRaftLimits,
) -> Result<(), CaptureRaftError> {
if request.vote.leader_id.node_id != source {
return Err(CaptureRaftError::Identity);
}
if request.entries.len() > limits.max_entries {
return Err(CaptureRaftError::RequestTooLarge);
}
for entry in &request.entries {
if let openraft::EntryPayload::Membership(value) = &entry.payload {
CaptureMembership::try_from(&openraft::StoredMembership::new(Some(entry.log_id), value.clone()))
.map_err(|_| CaptureRaftError::Protocol)?;
}
}
Ok(())
}
fn validate_snapshot(
request: &InstallSnapshotRequest<CaptureTypeConfig>,
source: u64,
limits: CaptureRaftLimits,
) -> Result<(), CaptureRaftError> {
if request.vote.leader_id.node_id != source
|| request.data.len() > limits.max_snapshot_chunk_bytes
|| request
.offset
.checked_add(u64::try_from(request.data.len()).map_err(|_| CaptureRaftError::Limits)?)
.is_none()
|| request.meta.snapshot_id.is_empty()
|| request.meta.snapshot_id.len() > 1024
{
return Err(CaptureRaftError::Limits);
}
CaptureMembership::try_from(&request.meta.last_membership).map_err(|_| CaptureRaftError::Protocol)?;
Ok(())
}
#[derive(Clone)]
pub struct CaptureRaftNetworkFactory {
local: NodeIdentity,
peers: Arc<BTreeMap<u64, CaptureNode>>,
limits: CaptureRaftLimits,
admission: Arc<tokio::sync::Semaphore>,
}
impl CaptureRaftNetworkFactory {
pub fn new(
local: NodeIdentity,
peers: BTreeMap<u64, CaptureNode>,
config: Arc<openraft::Config>,
limits: CaptureRaftLimits,
) -> Result<Self, CaptureRaftError> {
limits.validate()?;
peers_valid(local, &peers)?;
if usize::try_from(config.max_payload_entries).map_err(|_| CaptureRaftError::Limits)? > limits.max_entries {
return Err(CaptureRaftError::Limits);
}
if usize::try_from(config.snapshot_max_chunk_size).map_err(|_| CaptureRaftError::Limits)?
> limits.max_snapshot_chunk_bytes
{
return Err(CaptureRaftError::Limits);
}
Ok(Self {
local,
peers: Arc::new(peers),
limits,
admission: Arc::new(tokio::sync::Semaphore::new(limits.max_inflight)),
})
}
}
pub struct CaptureRaftPeer {
factory: CaptureRaftNetworkFactory,
target: u64,
node: CaptureNode,
valid: bool,
}
impl RaftNetworkFactory<CaptureTypeConfig> for CaptureRaftNetworkFactory {
type Network = CaptureRaftPeer;
async fn new_client(&mut self, target: u64, node: &CaptureNode) -> Self::Network {
CaptureRaftPeer {
factory: self.clone(),
target,
node: node.clone(),
valid: self.peers.get(&target) == Some(node),
}
}
}
impl CaptureRaftPeer {
async fn exchange<P: Serialize, R: Serialize + DeserializeOwned>(
&self,
method: Method,
payload: &P,
option: &RPCOption,
) -> Result<Result<R, RemoteFailure>, CaptureRaftError> {
if !self.valid {
return Err(CaptureRaftError::Identity);
}
let ttl = self.factory.limits.request_timeout.min(option.soft_ttl());
tokio::time::timeout(ttl, async {
let _permit = self
.factory
.admission
.clone()
.acquire_owned()
.await
.map_err(|_| CaptureRaftError::ChannelUnavailable)?;
let raw =
encode_request(self.factory.local, self.target, method, payload, self.factory.limits).map_err(
|error| match error {
CaptureRaftError::Limits => CaptureRaftError::RequestTooLarge,
error => error,
},
)?;
let mut request = Request::new(raw.clone());
http_auth::set_tonic_canonical_body_digest(&mut request, &canonical_request(&raw.body))?;
request.set_timeout(ttl);
let mut client = client::capture_raft_time_out_client(
self.node.rpc_uri(),
self.factory.limits.connect_timeout.min(ttl),
self.factory.limits.max_request_bytes + 64,
self.factory.limits.max_response_bytes + 128,
)
.await?;
let response = match method {
Method::AppendEntries => client.append_entries(request).await,
Method::Vote => client.vote(request).await,
Method::InstallSnapshot => client.install_snapshot(request).await,
}
.map_err(|status| CaptureRaftError::Status(Box::new(status)))?;
verify_response(&raw, response.into_inner(), self.factory.limits)
})
.await
.map_err(|_| CaptureRaftError::Deadline)?
}
fn network_error<E: std::error::Error>(
&self,
error: CaptureRaftError,
method: Method,
option: &RPCOption,
) -> RPCError<u64, CaptureNode, E> {
match error {
CaptureRaftError::Deadline => RPCError::Timeout(openraft::error::Timeout {
action: match method {
Method::AppendEntries => openraft::RPCTypes::AppendEntries,
Method::Vote => openraft::RPCTypes::Vote,
Method::InstallSnapshot => openraft::RPCTypes::InstallSnapshot,
},
id: self.factory.local.node_id,
target: self.target,
timeout: self.factory.limits.request_timeout.min(option.soft_ttl()),
}),
error @ CaptureRaftError::ChannelUnavailable => RPCError::Unreachable(Unreachable::new(&error)),
error => RPCError::Network(openraft::error::NetworkError::new(&error)),
}
}
fn append_error<E: std::error::Error>(
&self,
error: CaptureRaftError,
count: usize,
option: &RPCOption,
) -> RPCError<u64, CaptureNode, E> {
if matches!(error, CaptureRaftError::RequestTooLarge) && count > 1 {
let hint = u64::try_from((count / 2).max(1).min(self.factory.limits.max_entries)).unwrap_or(1);
return RPCError::PayloadTooLarge(openraft::error::PayloadTooLarge::new_entries_hint(hint));
}
self.network_error(error, Method::AppendEntries, option)
}
fn remote<E: std::error::Error>(&self, error: E) -> RPCError<u64, CaptureNode, E> {
RPCError::RemoteError(RemoteError::new_with_node(self.target, self.node.clone(), error))
}
}
impl RaftNetwork<CaptureTypeConfig> for CaptureRaftPeer {
async fn append_entries(
&mut self,
request: AppendEntriesRequest<CaptureTypeConfig>,
option: RPCOption,
) -> Result<AppendEntriesResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64>>> {
let count = request.entries.len();
validate_append(&request, self.factory.local.node_id, self.factory.limits)
.map_err(|error| self.append_error(error, count, &option))?;
match self
.exchange(Method::AppendEntries, &request, &option)
.await
.map_err(|error| self.append_error(error, count, &option))?
{
Ok(value) => Ok(value),
Err(error) => Err(self.remote(RaftError::Fatal(
restored_fatal(error).map_err(|error| self.network_error(error, Method::AppendEntries, &option))?,
))),
}
}
async fn vote(
&mut self,
request: VoteRequest<u64>,
option: RPCOption,
) -> Result<VoteResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64>>> {
if request.vote.leader_id.node_id != self.factory.local.node_id {
return Err(self.network_error(CaptureRaftError::Identity, Method::Vote, &option));
}
match self
.exchange(Method::Vote, &request, &option)
.await
.map_err(|error| self.network_error(error, Method::Vote, &option))?
{
Ok(value) => Ok(value),
Err(error) => Err(self.remote(RaftError::Fatal(
restored_fatal(error).map_err(|error| self.network_error(error, Method::Vote, &option))?,
))),
}
}
async fn install_snapshot(
&mut self,
request: InstallSnapshotRequest<CaptureTypeConfig>,
option: RPCOption,
) -> Result<InstallSnapshotResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64, InstallSnapshotError>>> {
validate_snapshot(&request, self.factory.local.node_id, self.factory.limits)
.map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?;
match self
.exchange(Method::InstallSnapshot, &request, &option)
.await
.map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?
{
Ok(value) => Ok(value),
Err(error) => {
let source = match error {
RemoteFailure::SnapshotMismatch { error } => RaftError::APIError(error),
error => RaftError::Fatal(
restored_fatal(error).map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?,
),
};
Err(self.remote(source))
}
}
}
}
fn auth_rejected(reason: &'static str) -> Status {
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
};
global_internode_metrics().record_rpc_auth_failure_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_OTHER,
INTERNODE_TRANSPORT_BACKEND_GRPC,
reason,
);
Status::unauthenticated("capture Raft authentication rejected")
}
File diff suppressed because it is too large Load Diff
+56
View File
@@ -535,6 +535,62 @@ impl tonic::service::Interceptor for TonicInterceptor {
}
}
pub(crate) type CaptureRaftClient = rustfs_protos::proto_gen::node_service::capture_raft_service_client::CaptureRaftServiceClient<
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
>;
/// Capture RPCs always perform the real authenticated epoch handshake, never seed replay state.
pub(crate) async fn capture_raft_time_out_client(
addr: &str,
connect_timeout: std::time::Duration,
request_max: usize,
response_max: usize,
) -> Result<CaptureRaftClient, super::capture_raft::CaptureRaftError> {
use super::capture_raft::CaptureRaftError;
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor()).with_rpc_audience(addr)?;
let channel = tokio::time::timeout(connect_timeout, async {
match runtime_sources::cached_node_channel(addr).await {
Some(channel) => Ok(channel),
None => create_new_channel(addr)
.await
.map_err(|_| CaptureRaftError::ChannelUnavailable),
}
})
.await
.map_err(|_| CaptureRaftError::Deadline)??;
let audience = interceptor
.replay_scope_audience()
.ok_or(CaptureRaftError::Authentication)?
.to_owned();
let authenticated = ReplayScopeChannel::new(channel, Some(audience.clone()));
let mut ping = NodeServiceClient::with_interceptor(authenticated.clone(), interceptor)
.max_encoding_message_size(1024)
.max_decoding_message_size(8192);
ping.ping(tonic::Request::new(rustfs_protos::proto_gen::node_service::PingRequest {
version: 1,
body: bytes::Bytes::new(),
}))
.await
.map_err(|status| CaptureRaftError::Status(Box::new(status)))?;
{
let peers = PEER_REPLAY_STATES.lock().map_err(|_| CaptureRaftError::Authentication)?;
let state = peers.get(&audience).ok_or(CaptureRaftError::Authentication)?;
if !matches!(state.cache_capability, Some(PeerReplayCapability::Capable { boot_epoch }) if Some(boot_epoch) == state.boot_epoch)
{
return Err(CaptureRaftError::Authentication);
}
}
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor()).with_rpc_audience(addr)?;
Ok(
rustfs_protos::proto_gen::node_service::capture_raft_service_client::CaptureRaftServiceClient::with_interceptor(
authenticated,
interceptor,
)
.max_encoding_message_size(request_max)
.max_decoding_message_size(response_max),
)
}
#[cfg(test)]
mod tests {
use super::*;
+3
View File
@@ -13,6 +13,9 @@
// limitations under the License.
pub(crate) mod background_monitor;
pub mod capture_raft;
#[cfg(test)]
mod capture_raft_tests;
pub(crate) mod client;
pub(crate) mod context_propagation;
pub(crate) mod http_auth;
+506 -5
View File
@@ -58,13 +58,15 @@ use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest,
MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest,
ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest,
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
MakeVolumesRequest, NativePreparationBeginRequest, NativePreparationFinalizeRequest, NativePreparationLocateRequest,
NativePreparationProbeRequest, NativePreparationRecoverRequest, PreparePartTransactionRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use sha2::{Digest, Sha256};
use std::{
future::Future,
io::Cursor,
@@ -176,6 +178,172 @@ fn attach_mutation_body_digest<T>(
crate::cluster::rpc::set_tonic_rolling_canonical_body_digest(request, &canonical_body).map_err(Error::other)
}
fn native_response_proof(body: std::result::Result<Vec<u8>, std::num::TryFromIntError>, proof: &[u8]) -> Result<()> {
let body = body.map_err(|_| Error::other("native preparation response is too large to authenticate"))?;
if proof.len() != 32 {
return Err(Error::other("native preparation response proof has invalid length"));
}
crate::cluster::rpc::verify_tonic_rpc_response_proof(&body, proof)
.map_err(|_| Error::other("native preparation response proof is invalid"))
}
fn native_response_error(error: Option<rustfs_protos::proto_gen::node_service::Error>) -> DiskError {
let error = DiskError::from(error.unwrap_or_default());
match error {
DiskError::FileCorrupt => DiskError::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, error)),
error => error,
}
}
fn native_uuid_field(value: &[u8], field: &str) -> Result<Uuid> {
let id = Uuid::from_slice(value).map_err(|_| Error::other(format!("native preparation returned invalid {field}")))?;
if id.is_nil() {
return Err(Error::other(format!("native preparation returned nil {field}")));
}
Ok(id)
}
fn native_version_field(value: &[u8]) -> Result<Uuid> {
Uuid::from_slice(value).map_err(|_| Error::other("native preparation returned invalid version id"))
}
fn native_object_length_from_metadata(input: &crate::disk::local::native_prepare::NativePrepareInput) -> Result<u64> {
let raw = rustfs_filemeta::FileMeta::load(&input.metadata).map_err(Error::other)?;
if raw.versions.len() != 1 {
return Err(Error::other("native preparation metadata must have one version"));
}
let info = rustfs_filemeta::get_file_info(
&input.metadata,
&input.bucket,
&input.key,
&input.version.to_string(),
rustfs_filemeta::FileInfoOpts {
data: true,
include_free_versions: false,
include_part_checksums: true,
},
)
.map_err(Error::other)?;
if info.version_id != Some(input.version) || info.deleted || info.mark_deleted || info.size < 0 {
return Err(Error::other("native preparation metadata identity mismatch"));
}
u64::try_from(info.size).map_err(Error::other)
}
fn validate_native_metadata_for_version(
metadata: &[u8],
expected_digest: Option<&[u8; 32]>,
bucket: &str,
key: &str,
version: Uuid,
) -> Result<()> {
validate_native_metadata(metadata, expected_digest)?;
let raw = rustfs_filemeta::FileMeta::load(metadata).map_err(Error::other)?;
if raw.versions.len() != 1 {
return Err(Error::other("native preparation metadata must have one version"));
}
let info = rustfs_filemeta::get_file_info(
metadata,
bucket,
key,
&version.to_string(),
rustfs_filemeta::FileInfoOpts {
data: true,
include_free_versions: false,
include_part_checksums: true,
},
)
.map_err(Error::other)?;
if info.version_id != Some(version) || info.deleted || info.mark_deleted || info.size < 0 {
return Err(Error::other("native preparation metadata identity mismatch"));
}
Ok(())
}
fn native_digest_field(value: &[u8], field: &str) -> Result<[u8; 32]> {
value
.try_into()
.map_err(|_| Error::other(format!("native preparation returned invalid {field}")))
}
fn native_receipt_from_wire(
receipt: rustfs_protos::proto_gen::node_service::NativePreparationReceipt,
expected_disk: Uuid,
expected_store: Uuid,
expected_preparation: Uuid,
expected_bucket: &str,
expected_key: &str,
expected_incarnation: Uuid,
expected_object: Option<(&[u8], u64)>,
expected_version: Option<Uuid>,
) -> Result<crate::disk::local::native_prepare::NativePreparationReceipt> {
let disk = native_uuid_field(&receipt.disk_id, "disk id")?;
let store = native_uuid_field(&receipt.store_id, "store id")?;
let deployment = native_uuid_field(&receipt.deployment_id, "deployment id")?;
let preparation = native_uuid_field(&receipt.preparation_id, "preparation id")?;
let version = native_version_field(&receipt.version_id)?;
let incarnation = native_uuid_field(&receipt.bucket_incarnation_id, "bucket incarnation")?;
if disk != expected_disk
|| store != expected_store
|| deployment != expected_store
|| preparation != expected_preparation
|| incarnation != expected_incarnation
|| receipt.bucket != expected_bucket
|| receipt.key != expected_key
|| expected_version.is_some_and(|expected| version != expected)
{
return Err(Error::other("native preparation receipt identity mismatch"));
}
let object_digest = native_digest_field(&receipt.object_digest, "object digest")?;
if expected_object.is_some_and(|(digest, length)| digest != object_digest.as_slice() || length != receipt.object_length) {
return Err(Error::other("native preparation receipt object mismatch"));
}
let set_digest = native_digest_field(&receipt.set_digest, "set digest")?;
let distribution_digest = native_digest_field(&receipt.distribution_digest, "distribution digest")?;
let metadata_digest = native_digest_field(&receipt.metadata_digest, "metadata digest")?;
let shard_digest = native_digest_field(&receipt.shard_digest, "shard digest")?;
let data_blocks = usize::try_from(receipt.data_blocks).map_err(Error::other)?;
let parity_blocks = usize::try_from(receipt.parity_blocks).map_err(Error::other)?;
let shard_index = usize::try_from(receipt.shard_index).map_err(Error::other)?;
if data_blocks == 0
|| parity_blocks == 0
|| data_blocks
.checked_add(parity_blocks)
.is_none_or(|total| shard_index == 0 || shard_index > total)
{
return Err(Error::other("native preparation receipt has invalid erasure geometry"));
}
Ok(crate::disk::local::native_prepare::NativePreparationReceipt {
preparation,
version,
disk,
set_digest,
distribution_digest,
metadata_digest,
shard_digest,
shard_length: receipt.shard_length,
bucket: receipt.bucket,
key: receipt.key,
deployment,
incarnation,
object_digest,
object_length: receipt.object_length,
data_blocks,
parity_blocks,
shard_index,
})
}
fn validate_native_metadata(metadata: &[u8], expected_digest: Option<&[u8; 32]>) -> Result<()> {
if metadata.len() > rustfs_protos::NATIVE_PREPARATION_METADATA_MAX_BYTES {
return Err(Error::other("native preparation metadata exceeds the protocol limit"));
}
if expected_digest.is_some_and(|expected| Sha256::digest(metadata).as_slice() != expected.as_slice()) {
return Err(Error::other("native preparation metadata digest mismatch"));
}
Ok(())
}
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
volume_infos
.into_iter()
@@ -896,6 +1064,339 @@ impl RemoteDisk {
.await
}
pub(crate) async fn native_begin_stream(
&self,
preparation: Uuid,
store_id: Uuid,
bucket: &str,
key: &str,
incarnation: Uuid,
) -> Result<()> {
let disk_id = self
.get_disk_id()
.await?
.filter(|id| !id.is_nil())
.ok_or(DiskError::InvalidPath)?;
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
return Err(DiskError::InvalidPath);
}
self.execute_with_timeout_for_op(
"native_preparation_begin",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(NativePreparationBeginRequest {
disk: self.disk_ref().await,
disk_id: disk_id.as_bytes().to_vec().into(),
store_id: store_id.as_bytes().to_vec().into(),
preparation_id: preparation.as_bytes().to_vec().into(),
bucket: bucket.to_owned(),
key: key.to_owned(),
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let canonical = rustfs_protos::canonical_native_preparation_begin_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical, "native_preparation_begin")?;
let canonical_request = request.get_ref().clone();
let response = client.native_preparation_begin(request).await?.into_inner();
native_response_proof(
rustfs_protos::canonical_native_preparation_begin_response_body(&canonical_request, &response),
&response.response_proof,
)?;
if !response.success {
return Err(native_response_error(response.error));
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
pub(crate) async fn native_finalize(
&self,
input: crate::disk::local::native_prepare::NativePrepareInput,
) -> Result<crate::disk::local::native_prepare::NativePreparationReceipt> {
let disk_id = self
.get_disk_id()
.await?
.filter(|id| !id.is_nil())
.ok_or(DiskError::InvalidPath)?;
if input.preparation.is_nil() || input.store_id.is_nil() || input.bucket_incarnation.is_nil() {
return Err(DiskError::InvalidPath);
}
validate_native_metadata(&input.metadata, None)?;
let expected_object_digest = input.object_digest;
let expected_object_length = native_object_length_from_metadata(&input)?;
let expected_version = input.version;
self.execute_with_timeout_for_op(
"native_preparation_finalize",
|| async move {
let mut client = self.get_client().await?;
let mut request = Request::new(NativePreparationFinalizeRequest {
disk: self.disk_ref().await,
disk_id: disk_id.as_bytes().to_vec().into(),
store_id: input.store_id.as_bytes().to_vec().into(),
preparation_id: input.preparation.as_bytes().to_vec().into(),
bucket: input.bucket.clone(),
key: input.key.clone(),
bucket_incarnation_id: input.bucket_incarnation.as_bytes().to_vec().into(),
object_digest: expected_object_digest.to_vec().into(),
object_length: expected_object_length,
version_id: expected_version.as_bytes().to_vec().into(),
metadata: input.metadata.clone(),
});
let canonical = rustfs_protos::canonical_native_preparation_finalize_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical, "native_preparation_finalize")?;
let canonical_request = request.get_ref().clone();
let response = client.native_preparation_finalize(request).await?.into_inner();
native_response_proof(
rustfs_protos::canonical_native_preparation_finalize_response_body(&canonical_request, &response),
&response.response_proof,
)?;
if !response.success {
return Err(native_response_error(response.error));
}
let receipt = response
.receipt
.ok_or_else(|| Error::other("native finalize response omitted its receipt"))?;
let receipt = native_receipt_from_wire(
receipt,
disk_id,
input.store_id,
input.preparation,
&input.bucket,
&input.key,
input.bucket_incarnation,
Some((&expected_object_digest, expected_object_length)),
Some(expected_version),
)?;
validate_native_metadata_for_version(
&input.metadata,
Some(&receipt.metadata_digest),
&input.bucket,
&input.key,
expected_version,
)?;
Ok(receipt)
},
get_max_timeout_duration(),
)
.await
}
pub(crate) async fn native_probe(
&self,
preparation: Uuid,
store_id: Uuid,
bucket: &str,
key: &str,
incarnation: Uuid,
) -> Result<crate::disk::local::native_prepare::NativePreparationProbe> {
let disk_id = self
.get_disk_id()
.await?
.filter(|id| !id.is_nil())
.ok_or(DiskError::InvalidPath)?;
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
return Err(DiskError::InvalidPath);
}
self.execute_with_timeout_for_op(
"native_preparation_probe",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(NativePreparationProbeRequest {
disk: self.disk_ref().await,
disk_id: disk_id.as_bytes().to_vec().into(),
store_id: store_id.as_bytes().to_vec().into(),
preparation_id: preparation.as_bytes().to_vec().into(),
bucket: bucket.to_owned(),
key: key.to_owned(),
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let canonical = rustfs_protos::canonical_native_preparation_probe_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical, "native_preparation_probe")?;
let canonical_request = request.get_ref().clone();
let response = client.native_preparation_probe(request).await?.into_inner();
native_response_proof(
rustfs_protos::canonical_native_preparation_probe_response_body(&canonical_request, &response),
&response.response_proof,
)?;
if !response.success {
return Err(native_response_error(response.error));
}
validate_native_metadata(&response.metadata, None)?;
if !response.exists {
if response.prepared
|| !response.metadata.is_empty()
|| !response.version_id.is_empty()
|| !response.object_digest.is_empty()
|| response.object_length != 0
|| !response.set_digest.is_empty()
|| !response.distribution_digest.is_empty()
{
return Err(Error::other("native probe absent response carries object state"));
}
return Ok(crate::disk::local::native_prepare::NativePreparationProbe::Absent);
}
let version = native_version_field(&response.version_id)?;
let object_digest = native_digest_field(&response.object_digest, "object digest")?;
let set_digest = native_digest_field(&response.set_digest, "set digest")?;
let distribution_digest = native_digest_field(&response.distribution_digest, "distribution digest")?;
if !response.metadata.is_empty() {
validate_native_metadata_for_version(&response.metadata, None, bucket, key, version)?;
}
Ok(crate::disk::local::native_prepare::NativePreparationProbe::Existing {
version,
prepared: response.prepared,
object_digest,
object_length: response.object_length,
set_digest,
distribution_digest,
metadata: (!response.metadata.is_empty()).then_some(response.metadata),
})
},
get_max_timeout_duration(),
)
.await
}
pub(crate) async fn native_recover(
&self,
preparation: Uuid,
store_id: Uuid,
bucket: &str,
key: &str,
incarnation: Uuid,
) -> Result<(crate::disk::local::native_prepare::NativePreparationReceipt, Bytes)> {
let disk_id = self
.get_disk_id()
.await?
.filter(|id| !id.is_nil())
.ok_or(DiskError::InvalidPath)?;
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
return Err(DiskError::InvalidPath);
}
self.execute_with_timeout_for_op(
"native_preparation_recover",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(NativePreparationRecoverRequest {
disk: self.disk_ref().await,
disk_id: disk_id.as_bytes().to_vec().into(),
store_id: store_id.as_bytes().to_vec().into(),
preparation_id: preparation.as_bytes().to_vec().into(),
bucket: bucket.to_owned(),
key: key.to_owned(),
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let canonical = rustfs_protos::canonical_native_preparation_recover_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical, "native_preparation_recover")?;
let canonical_request = request.get_ref().clone();
let response = client.native_preparation_recover(request).await?.into_inner();
native_response_proof(
rustfs_protos::canonical_native_preparation_recover_response_body(&canonical_request, &response),
&response.response_proof,
)?;
if !response.success {
return Err(native_response_error(response.error));
}
let receipt = response
.receipt
.ok_or_else(|| Error::other("native recover response omitted its receipt"))?;
let receipt =
native_receipt_from_wire(receipt, disk_id, store_id, preparation, bucket, key, incarnation, None, None)?;
validate_native_metadata_for_version(
&response.metadata,
Some(&receipt.metadata_digest),
bucket,
key,
receipt.version,
)?;
Ok((receipt, response.metadata))
},
get_max_timeout_duration(),
)
.await
}
pub(crate) async fn native_locate(
&self,
store_id: Uuid,
preparation: Uuid,
) -> Result<Option<crate::disk::local::native_prepare::NativePreparationDescription>> {
let disk_id = self
.get_disk_id()
.await?
.filter(|id| !id.is_nil())
.ok_or(DiskError::InvalidPath)?;
if preparation.is_nil() || store_id.is_nil() {
return Err(DiskError::InvalidPath);
}
self.execute_with_timeout_for_op(
"native_preparation_locate",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(NativePreparationLocateRequest {
disk: self.disk_ref().await,
disk_id: disk_id.as_bytes().to_vec().into(),
store_id: store_id.as_bytes().to_vec().into(),
preparation_id: preparation.as_bytes().to_vec().into(),
});
let canonical = rustfs_protos::canonical_native_preparation_locate_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical, "native_preparation_locate")?;
let canonical_request = request.get_ref().clone();
let response = client.native_preparation_locate(request).await?.into_inner();
native_response_proof(
rustfs_protos::canonical_native_preparation_locate_response_body(&canonical_request, &response),
&response.response_proof,
)?;
if !response.success {
return Err(native_response_error(response.error));
}
if !response.found {
if !response.bucket.is_empty()
|| !response.key.is_empty()
|| !response.bucket_incarnation_id.is_empty()
|| !response.metadata.is_empty()
|| response.prepared
{
return Err(Error::other("native locate absent response carries preparation state"));
}
return Ok(None);
}
if response.bucket.is_empty()
|| response.bucket.len() > 255
|| response.key.is_empty()
|| response.key.len() > 4096
|| preparation.is_nil()
{
return Err(Error::other("native locate response has invalid description"));
}
let incarnation = native_uuid_field(&response.bucket_incarnation_id, "bucket incarnation")?;
validate_native_metadata(&response.metadata, None)?;
if response.metadata.is_empty() {
return Err(Error::other("native locate response omitted metadata"));
}
let raw_metadata = rustfs_filemeta::FileMeta::load(&response.metadata)
.map_err(|_| Error::other("native locate response metadata is invalid"))?;
if raw_metadata.versions.len() != 1 {
return Err(Error::other("native locate metadata must have one version"));
}
let version = raw_metadata.versions[0].header.version_id.unwrap_or_else(Uuid::nil);
Ok(Some(crate::disk::local::native_prepare::NativePreparationDescription {
preparation,
bucket: response.bucket,
key: response.key,
incarnation,
version,
metadata: response.metadata,
prepared: response.prepared,
}))
},
get_max_timeout_duration(),
)
.await
}
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
+381
View File
@@ -449,6 +449,64 @@ where
Ok(data)
}
pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
where
S: EcstoreObjectIO,
{
let opts = ObjectOptions {
suppress_read_repair: true,
no_lock: false,
skip_verify_bitrot: false,
..Default::default()
};
crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
api, file, &opts, max_bytes,
))
.await
}
pub(crate) async fn read_config_observation_no_lock<S>(
api: Arc<S>,
file: &str,
max_bytes: usize,
guard: &crate::object_api::WriteCommitGuard,
) -> Result<(Vec<u8>, ObjectInfo)>
where
S: EcstoreObjectIO,
{
if !guard.matches(RUSTFS_META_BUCKET, file) {
return Err(Error::InvalidArgument(
RUSTFS_META_BUCKET.to_owned(),
file.to_owned(),
"locked observation requires a matching write owner".to_owned(),
));
}
let lock_lost_error = || Error::NamespaceLockQuorumUnavailable {
mode: "locked_config_observation",
bucket: RUSTFS_META_BUCKET.to_owned(),
object: file.to_owned(),
required: 1,
achieved: 0,
};
if guard.is_lock_lost() {
return Err(lock_lost_error());
}
let opts = ObjectOptions {
suppress_read_repair: true,
no_lock: true,
skip_verify_bitrot: false,
..Default::default()
};
let result = crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
api, file, &opts, max_bytes,
))
.await;
if guard.is_lock_lost() {
return Err(lock_lost_error());
}
result
}
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
where
S: EcstoreObjectIO,
@@ -5425,6 +5483,329 @@ mod tests {
assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
}
struct ObservationStream {
inner: Cursor<Vec<u8>>,
consumed: Arc<AtomicUsize>,
fail: bool,
eof_gate: Option<(Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>)>,
}
impl AsyncRead for ObservationStream {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
if self.fail {
return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
}
if self.inner.position() == u64::try_from(self.inner.get_ref().len()).expect("fixture length")
&& let Some((ready, release)) = self.eof_gate.as_mut()
{
if let Some(ready) = ready.take() {
ready.send(()).expect("EOF observer remains active");
}
match std::future::Future::poll(Pin::new(release), cx) {
Poll::Pending => return Poll::Pending,
Poll::Ready(result) => result.expect("EOF gate released"),
}
self.eof_gate = None;
}
let before = buf.filled().len();
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
result
}
}
struct ObservationStore {
body: Vec<u8>,
declared_size: i64,
error: Option<Error>,
stream_error: bool,
gets: AtomicUsize,
eof_gate: std::sync::Mutex<Option<(Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>)>>,
consumed: Arc<AtomicUsize>,
options: std::sync::Mutex<Option<(bool, bool, bool)>>,
writes: AtomicUsize,
}
impl Debug for ObservationStore {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ObservationStore").finish()
}
}
impl ObservationStore {
fn new(body: Vec<u8>, declared_size: i64) -> Self {
Self {
body,
declared_size,
error: None,
stream_error: false,
gets: AtomicUsize::new(0),
eof_gate: std::sync::Mutex::new(None),
consumed: Arc::new(AtomicUsize::new(0)),
options: std::sync::Mutex::new(None),
writes: AtomicUsize::new(0),
}
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
async fn get_object_reader(
&self,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
_headers: HeaderMap,
opts: &ObjectOptions,
) -> Result<GetObjectReader> {
assert_eq!(bucket, RUSTFS_META_BUCKET);
assert_eq!(object, "config/observation.bin");
self.gets.fetch_add(1, Ordering::SeqCst);
assert!(range.is_none());
*self.options.lock().expect("observation options") =
Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
if let Some(error) = &self.error {
return Err(error.clone());
}
Ok(GetObjectReader {
stream: Box::new(ObservationStream {
inner: Cursor::new(self.body.clone()),
consumed: self.consumed.clone(),
fail: self.stream_error,
eof_gate: self.eof_gate.lock().expect("EOF gate").take(),
}),
object_info: ObjectInfo {
size: self.declared_size,
etag: Some("observation-etag".to_string()),
..Default::default()
},
buffered_body: None,
body_source: Default::default(),
})
}
async fn put_object(
&self,
_bucket: &str,
_object: &str,
_data: &mut PutObjReader,
_opts: &ObjectOptions,
) -> Result<ObjectInfo> {
self.writes.fetch_add(1, Ordering::SeqCst);
Err(Error::other("observation must not write"))
}
}
#[tokio::test]
async fn locked_observation_preserves_options_and_typed_errors() {
let object = "config/observation.bin";
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-options".to_owned(), manager.clone()),
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
"locked-observation-options".to_owned(),
);
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
.await
.expect("matching real owner");
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
assert_eq!(result.is_ok(), true, "I06L_READ_FLAGS: {result:?}");
assert_eq!(
*store.options.lock().expect("options") == Some((true, true, false)),
true,
"I06L_READ_FLAGS"
);
assert_eq!(result.expect("raw observation").0, b"raw");
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
for (bucket, key) in [(RUSTFS_META_BUCKET, "config/other.bin"), ("wrong-bucket", object)] {
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-target".to_owned(), manager.clone()),
rustfs_lock::ObjectKey::new(bucket, key),
"locked-observation-target".to_owned(),
);
let wrong = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
.await
.expect("wrong-target real owner");
let store = Arc::new(ObservationStore::new(Vec::new(), 0));
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &wrong).await;
assert_eq!(matches!(result, Err(Error::InvalidArgument(..))), true, "I06L_TARGET_REJECTED");
assert_eq!(store.gets.load(Ordering::SeqCst) == 0, true, "I06L_TARGET_REJECTED");
}
for error in [
Error::FileNotFound,
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_owned(), object.to_owned()),
Error::ErasureReadQuorum,
] {
let expected = if error == Error::FileNotFound || matches!(error, Error::ObjectNotFound(..)) {
Error::ConfigNotFound
} else {
error.clone()
};
let mut fixture = ObservationStore::new(Vec::new(), 0);
fixture.error = Some(error);
let store = Arc::new(fixture);
assert_eq!(
super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
.await
.expect_err("typed error"),
expected
);
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
}
let mut fixture = ObservationStore::new(Vec::new(), 0);
fixture.stream_error = true;
let store = Arc::new(fixture);
let err = super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
.await
.expect_err("stream failure");
assert_eq!(err.to_string().contains("observation stream failed"), true, "I06L_TYPED_STREAM_ERROR");
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &guard).await;
assert_eq!(matches!(result, Err(Error::CorruptedFormat)), true, "I06L_STREAM_BOUND");
assert_eq!(store.consumed.load(Ordering::SeqCst) == 9, true, "I06L_STREAM_BOUND");
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn locked_observation_rejects_actual_owner_loss_before_and_after_read() {
let object = "config/observation.bin";
for after_read in [false, true] {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let client = Arc::new(RefreshControlledClient::new(manager));
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
rustfs_lock::NamespaceLock::new("locked-observation-loss".to_owned(), client.clone()),
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
"locked-observation-loss".to_owned(),
);
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
.await
.expect("real distributed owner");
let signal = guard.lock_lost_signal().expect("acquired distributed owner signal");
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
if after_read {
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
*store.eof_gate.lock().expect("EOF gate") = Some((Some(ready_tx), release_rx));
let mut read = Box::pin(super::read_config_observation_no_lock(store.clone(), object, 3, &guard));
tokio::time::timeout(std::time::Duration::from_secs(60), async {
tokio::select! {
ready = ready_rx => ready.expect("read reached EOF"),
result = &mut read => assert_eq!(false, true, "I06L_LOSS_AFTER_READ: returned before EOF gate: {result:?}"),
}
}).await.expect("EOF watchdog");
client.reject_refreshes();
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
.await
.expect("actual owner loss watchdog");
assert_eq!(guard.is_lock_lost(), true, "I06L_LOSS_AFTER_READ");
release_tx.send(()).expect("release EOF");
let result = tokio::time::timeout(std::time::Duration::from_secs(60), read)
.await
.expect("read watchdog");
assert_eq!(
matches!(
result,
Err(Error::NamespaceLockQuorumUnavailable {
mode: "locked_config_observation",
..
})
),
true,
"I06L_LOSS_AFTER_READ"
);
assert_eq!(store.gets.load(Ordering::SeqCst), 1);
} else {
client.reject_refreshes();
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
.await
.expect("actual owner loss watchdog");
assert_eq!(guard.is_lock_lost(), true, "I06L_LOSS_BEFORE_GET");
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
assert_eq!(
matches!(
result,
Err(Error::NamespaceLockQuorumUnavailable {
mode: "locked_config_observation",
..
})
),
true,
"I06L_LOSS_BEFORE_GET"
);
assert_eq!(store.gets.load(Ordering::SeqCst) == 0, true, "I06L_LOSS_BEFORE_GET");
}
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
}
}
#[tokio::test]
async fn observation_preserves_options_and_errors() {
for body in [Vec::new(), b"raw config".to_vec()] {
let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
let (observed, info) = result.expect("successful observation");
assert_eq!(observed, body);
assert_eq!(info.etag.as_deref(), Some("observation-etag"));
assert_eq!(
*store.options.lock().expect("options"),
Some((true, false, false)),
"I05_REPAIR_SUPPRESSED"
);
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
}
for (source, expected) in [
(Error::FileNotFound, Error::ConfigNotFound),
(
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
Error::ConfigNotFound,
),
(Error::ErasureReadQuorum, Error::ErasureReadQuorum),
] {
let mut fixture = ObservationStore::new(Vec::new(), 0);
fixture.error = Some(source);
let store = Arc::new(fixture);
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
assert_eq!(result.expect_err("read error must propagate"), expected);
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
}
let mut fixture = ObservationStore::new(Vec::new(), 0);
fixture.stream_error = true;
let store = Arc::new(fixture);
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
let error = result.expect_err("stream error must propagate");
assert!(error.to_string().contains("observation stream failed"));
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn observation_enforces_byte_limit() {
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
assert_eq!(result.is_err(), true, "I05_BOUND_ENFORCED");
assert_eq!(
result.expect_err("bounded observation error"),
Error::CorruptedFormat,
"I05_BOUND_ENFORCED"
);
assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
assert_eq!(result.expect("exact limit").0, vec![7; 8]);
assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
}
#[tokio::test]
async fn read_config_preserve_empty_distinguishes_empty_object() {
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
+190
View File
@@ -20,6 +20,7 @@ mod commit;
#[cfg(all(unix, test))]
#[path = "fd_cache_tests.rs"]
mod fd_cache_tests;
pub mod native_prepare;
mod replacement_lease;
#[cfg(any(target_os = "linux", test))]
#[path = "uring_driver_budget.rs"]
@@ -6866,6 +6867,28 @@ impl LocalDisk {
opt: &DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
#[cfg(unix)]
if let Some(native_path) = crate::disk::native_prepare_rpc::parse_native_stream_path(volume, path)? {
return match native_path.file {
crate::disk::native_prepare_rpc::NativeStreamFile::Staging(staging_id) => {
let root = self.io_root().to_path_buf();
tokio::task::spawn_blocking(move || {
native_prepare::remove_stream_staging_file(
&root,
native_path.preparation,
native_path.data_dir,
staging_id,
)
})
.await
.map_err(DiskError::from)?
.map_err(to_file_error)?;
Ok(())
}
crate::disk::native_prepare_rpc::NativeStreamFile::Part(_) => Err(DiskError::FileAccessDenied.into()),
};
}
let volume_dir = self.io_get_bucket_path(volume)?;
if !skip_access_checks(volume)
&& let Err(e) = cached_access(&volume_dir).await
@@ -10456,6 +10479,33 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
crate::hp_guard!("LocalDisk::rename_file");
#[cfg(unix)]
{
use crate::disk::native_prepare_rpc::{NativeStreamFile, parse_native_stream_path};
let src_native = parse_native_stream_path(src_volume, src_path)?;
let dst_native = parse_native_stream_path(dst_volume, dst_path)?;
if src_native.is_some() || dst_native.is_some() {
let (Some(src), Some(dst)) = (src_native, dst_native) else {
return Err(DiskError::InvalidPath.into());
};
let (NativeStreamFile::Staging(staging_id), NativeStreamFile::Part(part)) = (src.file, dst.file) else {
return Err(DiskError::InvalidPath.into());
};
if src_volume != dst_volume || src.preparation != dst.preparation || src.data_dir != dst.data_dir {
return Err(DiskError::InvalidPath.into());
}
let root = self.io_root().to_path_buf();
tokio::task::spawn_blocking(move || {
native_prepare::publish_stream_part(&root, src.preparation, src.data_dir, staging_id, part)
})
.await
.map_err(DiskError::from)?
.map_err(to_file_error)?;
return Ok(());
}
}
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
if !skip_access_checks(src_volume) {
@@ -10537,6 +10587,28 @@ impl DiskAPI for LocalDisk {
}
}
if let Some(native_path) = crate::disk::native_prepare_rpc::parse_native_stream_path(volume, path)? {
#[cfg(unix)]
{
use crate::disk::native_prepare_rpc::NativeStreamFile;
let root = self.io_root().to_path_buf();
let file = tokio::task::spawn_blocking(move || match native_path.file {
NativeStreamFile::Part(part) => {
native_prepare::open_stream_part(&root, native_path.preparation, native_path.data_dir, part)
}
NativeStreamFile::Staging(id) => {
native_prepare::open_stream_staging_file(&root, native_path.preparation, native_path.data_dir, id)
}
})
.await
.map_err(DiskError::from)?
.map_err(to_file_error)?;
return Ok(Box::new(tokio::fs::File::from_std(file)));
}
#[cfg(not(unix))]
return Err(DiskError::UnsupportedDisk);
}
self.io_backend
.open_write(volume, path, WriteMode::Truncate { size_hint: _file_size })
.await
@@ -11552,6 +11624,124 @@ mod test {
use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf};
use tracing_subscriber::fmt::MakeWriter;
#[cfg(unix)]
#[tokio::test]
#[serial_test::serial]
async fn native_stream_reserved_split_never_truncates_attested_shard() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
let (dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "native-split-regression";
let key = "object";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("real bucket");
let preparation = Uuid::new_v4();
drop(
store
.prepare_native_object(
bucket,
key,
preparation,
&mut crate::object_api::PutObjReader::from_vec(vec![51; 2 * 1024 * 1024 + 7]),
)
.await
.expect("actual erasure/bitrot preparation"),
);
let mut endpoint = Endpoint::try_from(dirs[0].path().to_str().expect("root")).expect("endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = LocalDisk::new(&endpoint, false).await.expect("actual LocalDisk");
let path = format!("native-preparations/{preparation}/{preparation}/part.1");
let full = dirs[0].path().join(RUSTFS_META_BUCKET).join(&path);
let before = std::fs::read(&full).expect("actual durable shard");
for (volume, candidate) in [
(
format!("{RUSTFS_META_BUCKET}/native-preparations/{preparation}"),
format!("{preparation}/part.1"),
),
(
format!("{RUSTFS_META_BUCKET}/native-preparations"),
format!("{preparation}/{preparation}/part.1"),
),
(RUSTFS_META_BUCKET.to_owned(), format!("./{path}")),
(RUSTFS_META_BUCKET.to_owned(), format!("junk/../{path}")),
(
format!("{RUSTFS_META_BUCKET}/junk/../native-preparations/{preparation}"),
format!("{preparation}/part.1"),
),
(
format!("junk/../{RUSTFS_META_BUCKET}"),
format!("native-preparations/{preparation}/{preparation}/part.1"),
),
(
RUSTFS_META_BUCKET.to_owned(),
format!("native-preparations//{preparation}/{preparation}/part.1"),
),
] {
assert!(
matches!(disk.create_file("", &volume, &candidate, 1).await, Err(DiskError::InvalidPath)),
"NATIVE_RESERVED_SPLIT_MUST_REJECT_BEFORE_TRUNCATE: {volume}/{candidate}"
);
assert_eq!(
std::fs::read(&full).expect("retained actual shard"),
before,
"NATIVE_RESERVED_SPLIT_MUST_PRESERVE_ATTESTED_SHARD"
);
}
let staged_id = Uuid::new_v4();
let staged_path = format!("native-preparations/{preparation}/{preparation}/.rustfs-put-{staged_id}");
let mut staged = disk
.create_file("", RUSTFS_META_BUCKET, &staged_path, 7)
.await
.expect("create private native staging file");
staged.write_all(b"staged!").await.expect("write staged payload");
staged.shutdown().await.expect("close staged payload");
disk.rename_file(
RUSTFS_META_BUCKET,
&staged_path,
RUSTFS_META_BUCKET,
&format!("native-preparations/{preparation}/{preparation}/part.2"),
)
.await
.expect("publish private staging file without ordinary volume rename");
assert_eq!(
std::fs::read(
dirs[0]
.path()
.join(RUSTFS_META_BUCKET)
.join(format!("native-preparations/{preparation}/{preparation}/part.2"))
)
.expect("read published private part"),
b"staged!"
);
let abandoned_id = Uuid::new_v4();
let abandoned_path = format!("native-preparations/{preparation}/{preparation}/.rustfs-put-{abandoned_id}");
let mut abandoned = disk
.create_file("", RUSTFS_META_BUCKET, &abandoned_path, 3)
.await
.expect("create private cleanup staging file");
abandoned.write_all(b"tmp").await.expect("write cleanup staging file");
abandoned.shutdown().await.expect("close cleanup staging file");
disk.delete(
RUSTFS_META_BUCKET,
&abandoned_path,
DeleteOptions {
immediate: true,
..Default::default()
},
)
.await
.expect("delete private staging file after failed publication");
assert!(!dirs[0].path().join(RUSTFS_META_BUCKET).join(&abandoned_path).exists());
assert!(store.recover_native_object(bucket, key, preparation).await.is_ok());
}
#[cfg(unix)]
#[tokio::test]
async fn read_dir_entries_skip_symlinks_without_truncating_listing() {
File diff suppressed because it is too large Load Diff
+147
View File
@@ -24,6 +24,7 @@ pub mod fs;
pub mod health_state;
mod io_admission;
pub mod local;
pub(crate) mod native_prepare_rpc;
pub mod os;
#[cfg(test)]
@@ -230,6 +231,152 @@ pub enum Disk {
}
impl Disk {
#[cfg(unix)]
pub(crate) async fn native_begin_stream(
&self,
preparation: Uuid,
store_id: Uuid,
bucket: &str,
key: &str,
incarnation: Uuid,
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
) -> Result<()> {
match self {
Disk::Local(local) => {
local
.get_disk()
.begin_native_stream(preparation, store_id, bucket, incarnation, fence)
.await
}
Disk::Remote(remote) => {
remote
.native_begin_stream(preparation, store_id, bucket, key, incarnation)
.await
}
}
}
#[cfg(not(unix))]
pub(crate) async fn native_begin_stream(
&self,
_preparation: Uuid,
_store_id: Uuid,
_bucket: &str,
_key: &str,
_incarnation: Uuid,
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
) -> Result<()> {
Err(DiskError::UnsupportedDisk)
}
#[cfg(unix)]
pub(crate) async fn native_finalize(
&self,
input: local::native_prepare::NativePrepareInput,
) -> Result<local::native_prepare::NativePreparationReceipt> {
match self {
Disk::Local(local) => local.get_disk().prepare_native(input).await,
Disk::Remote(remote) => remote.native_finalize(input).await,
}
}
#[cfg(not(unix))]
pub(crate) async fn native_finalize(
&self,
_input: local::native_prepare::NativePrepareInput,
) -> Result<local::native_prepare::NativePreparationReceipt> {
Err(DiskError::UnsupportedDisk)
}
#[cfg(unix)]
pub(crate) async fn native_recover(
&self,
preparation: Uuid,
store_id: Uuid,
bucket: &str,
key: &str,
incarnation: Uuid,
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
) -> Result<(local::native_prepare::NativePreparationReceipt, Bytes)> {
match self {
Disk::Local(local) => {
local
.get_disk()
.recover_native_with_metadata(preparation, store_id, bucket, incarnation, fence)
.await
}
Disk::Remote(remote) => remote.native_recover(preparation, store_id, bucket, key, incarnation).await,
}
}
#[cfg(not(unix))]
pub(crate) async fn native_recover(
&self,
_preparation: Uuid,
_store_id: Uuid,
_bucket: &str,
_key: &str,
_incarnation: Uuid,
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
) -> Result<(local::native_prepare::NativePreparationReceipt, Bytes)> {
Err(DiskError::UnsupportedDisk)
}
#[cfg(unix)]
pub(crate) async fn native_probe(
&self,
preparation: Uuid,
store_id: Uuid,
bucket: &str,
key: &str,
incarnation: Uuid,
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
) -> Result<local::native_prepare::NativePreparationProbe> {
match self {
Disk::Local(local) => {
local
.get_disk()
.probe_native(preparation, store_id, bucket, key, incarnation, fence)
.await
}
Disk::Remote(remote) => remote.native_probe(preparation, store_id, bucket, key, incarnation).await,
}
}
#[cfg(not(unix))]
pub(crate) async fn native_probe(
&self,
_preparation: Uuid,
_store_id: Uuid,
_bucket: &str,
_key: &str,
_incarnation: Uuid,
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
) -> Result<local::native_prepare::NativePreparationProbe> {
Err(DiskError::UnsupportedDisk)
}
#[cfg(unix)]
pub(crate) async fn native_locate(
&self,
store_id: Uuid,
preparation: Uuid,
) -> Result<Option<local::native_prepare::NativePreparationDescription>> {
match self {
Disk::Local(local) => local.get_disk().locate_native_preparation(store_id, preparation).await,
Disk::Remote(remote) => remote.native_locate(store_id, preparation).await,
}
}
#[cfg(not(unix))]
pub(crate) async fn native_locate(
&self,
_store_id: Uuid,
_preparation: Uuid,
) -> Result<Option<local::native_prepare::NativePreparationDescription>> {
Err(DiskError::UnsupportedDisk)
}
pub(crate) async fn set_disk_id_state(&self, id: Option<Uuid>) -> Result<()> {
match self {
Disk::Local(local_disk) => {
@@ -0,0 +1,145 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::{
RUSTFS_META_BUCKET,
error::{DiskError, Result},
};
use rustfs_utils::path::path_join_buf;
use uuid::Uuid;
const NATIVE_STREAM_PREFIX: &str = "native-preparations/";
const NATIVE_STREAM_ROOT: &str = "native-preparations";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum NativeStreamFile {
Part(usize),
Staging(Uuid),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct NativeStreamPath {
pub(crate) preparation: Uuid,
pub(crate) data_dir: Uuid,
pub(crate) file: NativeStreamFile,
}
fn targets_native_stream_namespace(volume: &str, path: &str) -> bool {
if !volume.contains(RUSTFS_META_BUCKET) && !path.contains(NATIVE_STREAM_ROOT) {
return false;
}
let normalized = path_join_buf(&[volume, path]);
let prefix = format!("{RUSTFS_META_BUCKET}/{NATIVE_STREAM_PREFIX}");
normalized == format!("{RUSTFS_META_BUCKET}/{NATIVE_STREAM_ROOT}") || normalized.starts_with(&prefix)
}
/// Recognize only the reserved per-preparation shard path used by the native
/// stream writer. Malformed paths under that namespace must not fall through to
/// the general disk writer.
pub(crate) fn parse_native_stream_path(volume: &str, path: &str) -> Result<Option<NativeStreamPath>> {
if !targets_native_stream_namespace(volume, path) {
return Ok(None);
}
if volume != RUSTFS_META_BUCKET {
return Err(DiskError::InvalidPath);
}
if !path.starts_with(NATIVE_STREAM_PREFIX) {
return Err(DiskError::InvalidPath);
}
let mut components = path.split('/');
if components.next() != Some("native-preparations") {
return Err(DiskError::InvalidPath);
}
let preparation = components
.next()
.and_then(|value| Uuid::parse_str(value).ok())
.filter(|value| !value.is_nil())
.ok_or(DiskError::InvalidPath)?;
let data_dir = components
.next()
.and_then(|value| Uuid::parse_str(value).ok())
.filter(|value| !value.is_nil())
.ok_or(DiskError::InvalidPath)?;
let file_name = components.next().ok_or(DiskError::InvalidPath)?;
let file = file_name
.strip_prefix("part.")
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.filter(|part| file_name == format!("part.{part}"))
.map(NativeStreamFile::Part)
.or_else(|| {
file_name
.strip_prefix(".rustfs-put-")
.and_then(|value| Uuid::parse_str(value).ok())
.filter(|value| !value.is_nil())
.filter(|id| file_name == format!(".rustfs-put-{id}"))
.map(NativeStreamFile::Staging)
});
if components.next().is_some()
|| file.is_none()
|| path != format!("native-preparations/{preparation}/{data_dir}/{file_name}")
{
return Err(DiskError::InvalidPath);
}
Ok(Some(NativeStreamPath {
preparation,
data_dir,
file: file.expect("validated native stream filename"),
}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn native_stream_path_requires_canonical_identity_and_part() {
let preparation = Uuid::from_u128(1);
let data_dir = Uuid::from_u128(2);
let valid = format!("native-preparations/{preparation}/{data_dir}/part.1");
assert!(
parse_native_stream_path(RUSTFS_META_BUCKET, &valid)
.expect("valid path")
.is_some()
);
let staging = format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-{data_dir}");
assert!(
parse_native_stream_path(RUSTFS_META_BUCKET, &staging)
.expect("valid staging path")
.is_some()
);
assert!(parse_native_stream_path("bucket", &valid).expect("other volume").is_none());
assert!(
parse_native_stream_path(RUSTFS_META_BUCKET, "ordinary/object")
.expect("ordinary path")
.is_none()
);
for invalid in [
"native-preparations/00000000-0000-0000-0000-000000000000/00000000-0000-0000-0000-000000000002/part.1".to_owned(),
format!("native-preparations/{preparation}/{data_dir}/part.0"),
format!("native-preparations/{preparation}/{data_dir}/part.01"),
format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-{}", Uuid::nil()),
format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-not-a-uuid"),
format!("native-preparations/{preparation}/{data_dir}/part.1/extra"),
format!("native-preparations/{preparation}/../part.1"),
] {
assert!(matches!(
parse_native_stream_path(RUSTFS_META_BUCKET, &invalid),
Err(DiskError::InvalidPath)
));
}
}
}
@@ -68,7 +68,7 @@ impl WriteCommitGuard {
self.guards.iter().any(|guard| guard.is_lock_lost() || guard.is_released())
}
fn matches(&self, bucket: &str, object: &str) -> bool {
pub(crate) fn matches(&self, bucket: &str, object: &str) -> bool {
self.bucket.as_ref() == bucket && self.object.as_ref() == object
}
}
+306
View File
@@ -878,6 +878,7 @@ pub(crate) use ops::heal::AbsenceProofRequest;
#[cfg(test)]
pub(crate) use ops::heal::DanglingDeleteFailure;
pub(crate) use ops::heal::HealedObjectAbsence;
pub(crate) use ops::native_prepare::same_identity;
#[cfg(test)]
pub(crate) use ops::hermetic_set_disks_isolated;
@@ -1095,6 +1096,311 @@ mod prepared_get_object_metadata_tests {
use http::HeaderMap;
use tokio::io::AsyncReadExt;
const OBSERVATION_HOOK_BODY_LEN: usize = 1024 * 1024 + 1;
struct ObservationBodyHook {
object: &'static str,
calls: Arc<AtomicU64>,
body: bytes::Bytes,
}
#[async_trait::async_trait]
impl crate::object_api::GetObjectBodyCacheHook for ObservationBodyHook {
async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<bytes::Bytes> {
if bucket == crate::disk::RUSTFS_META_BUCKET && object == self.object {
assert_eq!(info.size, i64::try_from(self.body.len()).expect("hook fixture size"));
self.calls.fetch_add(1, Ordering::SeqCst);
Some(self.body.clone())
} else {
None
}
}
}
struct ObservationHookGuard;
impl Drop for ObservationHookGuard {
fn drop(&mut self) {
crate::object_api::unregister_get_object_body_cache_hook();
}
}
async fn ordinary_observation_get(set: &SetDisks, object: &str) -> Vec<u8> {
let mut reader = set
.get_object_reader(crate::disk::RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("ordinary GET positive control");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("ordinary GET body");
bytes
}
fn observation_register_hook(object: &'static str, body: Vec<u8>) -> (ObservationHookGuard, Arc<AtomicU64>) {
let calls = Arc::new(AtomicU64::new(0));
crate::object_api::register_get_object_body_cache_hook(Arc::new(ObservationBodyHook {
object,
calls: calls.clone(),
body: bytes::Bytes::from(body),
}));
(ObservationHookGuard, calls)
}
#[tokio::test]
#[serial_test::serial(body_cache_hook)]
async fn locked_observation_uses_real_outer_guard() {
let object = "config/locked-observation-real.bin";
let payload = vec![b'p'; OBSERVATION_HOOK_BODY_LEN];
let (_dirs, set) = make_local_set_disks(4, 2).await;
let mut reader = PutObjReader::from_vec(payload.clone());
let written = set
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
.await
.expect("persist locked observation fixture");
let (_hook, calls) = observation_register_hook(object, vec![b'x'; payload.len()]);
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
let namespace = set
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
.await
.expect("namespace");
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
.await
.expect("matching real owner");
let mut contender = Box::pin(crate::object_api::WriteCommitGuard::acquire(
&namespace,
std::time::Duration::from_secs(30),
));
assert_eq!(futures::poll!(contender.as_mut()).is_pending(), true, "I06L_OWNER_RETAINED");
let result = tokio::time::timeout(
std::time::Duration::from_secs(10),
crate::config::com::read_config_observation_no_lock(set.clone(), object, payload.len(), &guard),
)
.await
.expect("locked observation hang watchdog");
assert_eq!(result.is_ok(), true, "I06L_NO_NESTED_LOCK: {result:?}");
let (bytes, info) = result.expect("locked observation");
assert_eq!(bytes == payload, true, "I06L_HOOK_BYPASS");
assert_eq!(calls.load(Ordering::SeqCst) == 0, true, "I06L_HOOK_BYPASS");
assert_eq!(info.size, written.size);
assert_eq!(info.etag, written.etag);
assert_eq!(info.data_dir, written.data_dir);
assert_eq!(info.name, object);
assert_eq!(futures::poll!(contender.as_mut()).is_pending(), true, "I06L_OWNER_RETAINED");
drop(guard);
let next = tokio::time::timeout(std::time::Duration::from_secs(10), contender)
.await
.expect("owner release watchdog")
.expect("competing owner acquired");
drop(next);
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
#[serial_test::serial(body_cache_hook)]
async fn locked_observation_preserves_empty_missing_and_bound() {
let (_dirs, set) = make_local_set_disks(4, 2).await;
for (object, body) in [
("config/locked-empty.bin", Vec::new()),
("config/locked-bound.bin", b"raw".to_vec()),
] {
let mut reader = PutObjReader::from_vec(body.clone());
let written = set
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
.await
.expect("persist boundary fixture");
let namespace = set
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
.await
.expect("namespace");
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
.await
.expect("matching boundary owner");
let result = crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len(), &guard).await;
assert_eq!(result.is_ok(), true, "I06L_EMPTY_DISTINCT: {result:?}");
let (bytes, info) = result.expect("boundary observation");
assert_eq!(bytes, body);
assert_eq!(info.size, written.size);
if !body.is_empty() {
let result =
crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len() - 1, &guard).await;
assert_eq!(matches!(result, Err(Error::CorruptedFormat)), true, "I06L_BOUND_ENFORCED");
}
}
let object = "config/locked-absent.bin";
let namespace = set
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
.await
.expect("namespace");
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
.await
.expect("matching missing owner");
let result = crate::config::com::read_config_observation_no_lock(set, object, 0, &guard).await;
assert_eq!(matches!(result, Err(Error::ConfigNotFound)), true, "I06L_EMPTY_DISTINCT");
}
#[tokio::test]
#[serial_test::serial(body_cache_hook)]
async fn observation_real_backend_boundaries() {
let object = "config/observation-real.bin";
let payload = b"persisted config";
let (_dirs, set) = make_local_set_disks(4, 2).await;
let mut reader = PutObjReader::from_vec(payload.to_vec());
let written = set
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
.await
.expect("persist real config fixture");
let result = crate::config::com::read_config_observation(set.clone(), object, payload.len()).await;
assert_eq!(result.is_ok(), true, "I05_OBSERVATION_SUCCESS: {result:?}");
let (bytes, info) = result.expect("real observation success");
assert_eq!(bytes, payload);
assert_eq!(info.size, written.size);
assert_eq!(info.etag, written.etag);
assert_eq!(info.data_dir, written.data_dir);
assert_eq!(info.name, object);
let over = crate::config::com::read_config_observation(set.clone(), object, payload.len() - 1).await;
assert_eq!(over.is_err(), true, "I05_BOUND_ENFORCED");
assert_eq!(over.expect_err("over limit must fail"), Error::CorruptedFormat, "I05_BOUND_ENFORCED");
let absent = crate::config::com::read_config_observation(set.clone(), "config/observation-absent.bin", 32).await;
assert_eq!(absent.expect_err("absent config"), Error::ConfigNotFound);
let empty_object = "config/observation-empty.bin";
let mut empty = PutObjReader::from_vec(Vec::new());
set.put_object(crate::disk::RUSTFS_META_BUCKET, empty_object, &mut empty, &ObjectOptions::default())
.await
.expect("persist empty config");
let empty = crate::config::com::read_config_observation(set, empty_object, 0).await;
assert!(empty.is_ok(), "existing empty config must succeed: {empty:?}");
let (bytes, info) = empty.expect("empty observation");
assert!(bytes.is_empty());
assert_eq!(info.size, 0);
}
#[tokio::test]
#[serial_test::serial(body_cache_hook)]
async fn observation_bypasses_registered_body_hook() {
let object = "config/observation-hook.bin";
let (_dirs, set) = make_local_set_disks(4, 2).await;
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
.await
.expect("persist real config fixture");
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
let result = crate::config::com::read_config_observation(set, object, OBSERVATION_HOOK_BODY_LEN).await;
assert_eq!(result.is_ok(), true, "I05_HOOK_BYPASS: {result:?}");
assert_eq!(
result.expect("observed persisted body").0,
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
"I05_HOOK_BYPASS"
);
assert_eq!(calls.load(Ordering::SeqCst), 0, "I05_HOOK_BYPASS");
}
#[tokio::test]
#[serial_test::serial(body_cache_hook)]
async fn observation_hook_scope_restores_after_error() {
let object = "config/observation-hook-error.bin";
let (_dirs, set) = make_local_set_disks(4, 2).await;
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
.await
.expect("persist real config fixture");
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
let result = crate::config::com::read_config_observation(set.clone(), object, OBSERVATION_HOOK_BODY_LEN - 1).await;
assert_eq!(result.expect_err("bounded observation error"), Error::CorruptedFormat);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
struct PendingObservationStore {
set: Arc<SetDisks>,
ready: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
}
impl std::fmt::Debug for PendingObservationStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PendingObservationStore").finish()
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::object::ObjectIO for PendingObservationStore {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
async fn get_object_reader(
&self,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
headers: HeaderMap,
opts: &ObjectOptions,
) -> Result<GetObjectReader> {
let mut reader = self.set.get_object_reader(bucket, object, range, headers, opts).await?;
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("pending producer real GET");
assert_eq!(
body,
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
"I05_HOOK_BYPASS: producer must enter its own scope"
);
self.ready
.lock()
.expect("pending signal")
.take()
.expect("one GET")
.send(())
.expect("producer entry signal");
std::future::pending::<Result<GetObjectReader>>().await
}
async fn put_object(
&self,
_bucket: &str,
_object: &str,
_data: &mut PutObjReader,
_opts: &ObjectOptions,
) -> Result<ObjectInfo> {
assert!(false, "observation must not PUT");
Err(Error::other("unexpected observation PUT"))
}
}
#[tokio::test]
#[serial_test::serial(body_cache_hook)]
async fn observation_hook_scope_restores_after_cancel() {
let object = "config/observation-hook-cancel.bin";
let (_dirs, set) = make_local_set_disks(4, 2).await;
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
.await
.expect("persist real config fixture");
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
let adapter = Arc::new(PendingObservationStore {
set: set.clone(),
ready: std::sync::Mutex::new(Some(ready_tx)),
});
let mut scope = Box::pin(crate::config::com::read_config_observation(adapter, object, OBSERVATION_HOOK_BODY_LEN));
tokio::select! {
result = ready_rx => { result.expect("producer reached pending GET"); }
result = &mut scope => { assert!(false, "producer must remain pending until cancelled: {result:?}"); }
}
assert_eq!(calls.load(Ordering::SeqCst), 0);
drop(scope);
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
+1
View File
@@ -24,6 +24,7 @@ pub(crate) mod heal_walk;
pub(crate) mod list;
pub(crate) mod locking;
pub(crate) mod multipart;
pub(crate) mod native_prepare;
pub(crate) mod object;
#[cfg(test)]
@@ -0,0 +1,783 @@
//! Encode native candidates privately; only LocalDisk can attest durable shards.
use crate::{
disk::{
Disk,
error::Error as DiskError,
local::native_prepare::{NativePreparationProbe, NativePreparationReceipt, NativePrepareInput},
},
erasure::coding::{BitrotReader, Erasure, encode::IntegrityEncodeMode},
error::{Error, Result},
object_api::ObjectLockConfigSnapshot,
set_disk::{SetDisks, resolve_write_layout},
};
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, FileInfoOpts, FileMeta};
use sha2::{Digest, Sha256};
use std::{io::Cursor, sync::Arc};
use tokio::sync::mpsc;
use uuid::Uuid;
pub(crate) fn same_identity(a: &NativePreparationReceipt, b: &NativePreparationReceipt) -> bool {
a.preparation_id() == b.preparation_id()
&& a.version_id() == b.version_id()
&& a.bucket() == b.bucket()
&& a.key() == b.key()
&& a.deployment_id() == b.deployment_id()
&& a.bucket_incarnation() == b.bucket_incarnation()
&& a.object_digest() == b.object_digest()
&& a.object_length() == b.object_length()
&& a.set_digest() == b.set_digest()
&& a.distribution_digest() == b.distribution_digest()
&& a.erasure_geometry().0 == b.erasure_geometry().0
&& a.erasure_geometry().1 == b.erasure_geometry().1
}
pub(super) struct DigestReader<R> {
inner: R,
digest: Sha256,
}
impl<R> DigestReader<R> {
pub(super) fn new(inner: R) -> Self {
Self {
inner,
digest: Sha256::new(),
}
}
pub(super) fn into_inner_and_digest(self) -> (R, [u8; 32]) {
(self.inner, self.digest.finalize().into())
}
}
impl<R: tokio::io::AsyncRead + Unpin> tokio::io::AsyncRead for DigestReader<R> {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
let before = buf.filled().len();
let result = std::pin::Pin::new(&mut self.inner).poll_read(cx, buf);
if matches!(result, std::task::Poll::Ready(Ok(()))) {
self.digest.update(&buf.filled()[before..]);
}
result
}
}
struct DigestSink {
digest: Sha256,
length: usize,
}
impl tokio::io::AsyncWrite for DigestSink {
fn poll_write(
mut self: std::pin::Pin<&mut Self>,
_: &mut std::task::Context<'_>,
bytes: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
match self.length.checked_add(bytes.len()) {
Some(length) => {
self.length = length;
self.digest.update(bytes);
std::task::Poll::Ready(Ok(bytes.len()))
}
None => std::task::Poll::Ready(Err(std::io::Error::other("native content length overflow"))),
}
}
fn poll_flush(self: std::pin::Pin<&mut Self>, _: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
fn poll_shutdown(self: std::pin::Pin<&mut Self>, _: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
}
impl SetDisks {
fn native_expected_set_digest(&self, store_id: Uuid) -> Result<[u8; 32]> {
let set = self
.format
.erasure
.sets
.get(self.set_index)
.ok_or_else(|| Error::other("native target set missing"))?;
if self.format.id != store_id
|| set.len() != self.set_drive_count
|| set.iter().any(Uuid::is_nil)
|| set.iter().enumerate().any(|(slot, id)| set[slot + 1..].contains(id))
{
return Err(Error::other("invalid native target physical set"));
}
Ok(Sha256::digest(serde_json::to_vec(set)?).into())
}
/// Attest already encoded private shards while the caller owns the actual PUT commit guards.
pub(crate) async fn attest_native_stream(
&self,
store_id: Uuid,
bucket: &str,
key: &str,
preparation: Uuid,
incarnation: Uuid,
candidates: Vec<(crate::disk::DiskStore, FileInfo)>,
object_digest: [u8; 32],
object_length: u64,
fence: Arc<ObjectLockConfigSnapshot>,
) -> Result<crate::store::native_prepare::NativeQuorumPreparation> {
self.attest_native_stream_owned(
store_id,
bucket,
key,
preparation,
incarnation,
candidates,
object_digest,
object_length,
fence,
None,
)
.await
}
async fn attest_native_stream_owned(
&self,
store_id: Uuid,
bucket: &str,
key: &str,
preparation: Uuid,
incarnation: Uuid,
candidates: Vec<(crate::disk::DiskStore, FileInfo)>,
object_digest: [u8; 32],
object_length: u64,
fence: Arc<ObjectLockConfigSnapshot>,
operation_owner: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
) -> Result<crate::store::native_prepare::NativeQuorumPreparation> {
let expected_set = self.native_expected_set_digest(store_id)?;
if !fence.is_valid_for_destructive_put(store_id, bucket, incarnation) {
return Err(Error::other("native adoption bucket fence mismatch"));
}
let layout = resolve_write_layout(
&self.storage_class_config_snapshot(),
self.pool_index,
self.set_drive_count,
self.default_parity_count,
None,
false,
)?;
let (send, mut receive) = mpsc::unbounded_channel();
for (disk, info) in candidates {
if info.size != i64::try_from(object_length).map_err(Error::other)?
|| (info.name != key && info.name != format!("{bucket}/{key}"))
|| info.inline_data()
|| info.deleted
|| info.mark_deleted
{
return Err(Error::other("native adoption metadata identity mismatch"));
}
let version = info
.version_id
.filter(|id| !id.is_nil())
.ok_or_else(|| Error::other("native adoption version missing"))?;
let mut metadata = FileMeta::new();
metadata.add_version(info)?;
let input = NativePrepareInput {
preparation,
store_id,
bucket: bucket.into(),
bucket_incarnation: incarnation,
object_digest,
key: key.into(),
version,
metadata: metadata.marshal_msg()?.into(),
bucket_fence: fence.clone(),
};
let send = send.clone();
let owner = operation_owner.clone();
tokio::spawn(async move {
let result = match (disk.as_ref(), owner) {
(Disk::Local(local), Some(owner)) => local.get_disk().prepare_native_with_owner(input, owner).await,
(_, owner) => {
let _owner = owner;
disk.native_finalize(input).await
}
};
let _ = send.send(result);
});
}
drop(send);
let mut receipts = Vec::new();
while let Some(result) = receive.recv().await {
let receipt = match result {
Ok(receipt) => receipt,
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData => return Err(error.into()),
Err(_) => continue,
};
if receipt.set_digest() != &expected_set
|| receipts.first().is_some_and(|old| !same_identity(old, &receipt))
|| receipts
.iter()
.any(|old: &NativePreparationReceipt| old.disk_id() == receipt.disk_id())
{
return Err(Error::other("native adoption quorum identity mismatch"));
}
receipts.push(receipt);
}
crate::store::native_prepare::NativeQuorumPreparation::from_receipts(layout.write_quorum, receipts)
}
pub(crate) async fn prepare_native_stream(
&self,
store_id: Uuid,
bucket: &str,
key: &str,
preparation: Uuid,
incarnation: Uuid,
data: &mut crate::object_api::PutObjReader,
fence: Arc<ObjectLockConfigSnapshot>,
operation_owner: Arc<rustfs_lock::NamespaceLockGuard>,
) -> Result<(usize, Vec<NativePreparationReceipt>)> {
use crate::disk::RUSTFS_META_BUCKET;
let disks = self.get_disks_internal().await;
let layout = resolve_write_layout(
&self.storage_class_config_snapshot(),
self.pool_index,
disks.len(),
self.default_parity_count,
None,
false,
)?;
let mut info = FileInfo::new(key, layout.data_drives, layout.parity_drives);
info.name = key.to_owned();
info.version_id = Some(preparation);
info.data_dir = Some(preparation);
info.size = data.size();
info.versioned = true;
info.mod_time = Some(time::OffsetDateTime::now_utc());
let erasure = Arc::new(Erasure::try_new_with_options(
info.erasure.data_blocks,
info.erasure.parity_blocks,
info.erasure.block_size,
false,
)?);
let mut shuffled = Self::shuffle_disks_owned(disks.clone(), &info.erasure.distribution);
let path = format!("native-preparations/{preparation}/{preparation}/part.1");
let mut writers = Vec::with_capacity(shuffled.len());
for disk in &mut shuffled {
let result = match disk {
Some(disk) => {
if let Err(error) = disk
.native_begin_stream(preparation, store_id, bucket, key, incarnation, fence.clone())
.await
{
if matches!(&error, DiskError::Io(error) if error.kind() == std::io::ErrorKind::InvalidData) {
return Err(error.into());
}
writers.push(None);
continue;
}
crate::io_support::bitrot::create_bitrot_writer(
false,
Some(disk),
RUSTFS_META_BUCKET,
&path,
erasure.shard_file_size(info.size),
erasure.shard_size(),
rustfs_utils::HashAlgorithm::HighwayHash256S,
)
.await
}
None => Err(DiskError::DiskNotFound),
};
match result {
Ok(writer) => writers.push(Some(writer)),
Err(_) => {
*disk = None;
writers.push(None);
}
}
}
let stream = std::mem::replace(
&mut data.stream,
rustfs_rio::HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
);
let reader = DigestReader::new(stream);
let (reader, size, _, _) = erasure
.encode_with_shard_integrity(reader, &mut writers, layout.write_quorum, 1, IntegrityEncodeMode::Streaming, false)
.await?;
let (stream, digest) = reader.into_inner_and_digest();
data.stream = stream;
if info.size >= 0 && i64::try_from(size).map_err(Error::other)? != info.size {
return Err(Error::other("native stream length mismatch"));
}
info.size = i64::try_from(size).map_err(Error::other)?;
info.add_object_part(1, String::new(), size, None, info.size, None, None);
let mut candidates = Vec::new();
for (slot, disk) in disks.into_iter().enumerate() {
let index = *info
.erasure
.distribution
.get(slot)
.ok_or_else(|| Error::other("native stream distribution missing"))?;
if writers
.get(
index
.checked_sub(1)
.ok_or_else(|| Error::other("invalid native stream index"))?,
)
.and_then(Option::as_ref)
.is_none()
{
continue;
}
let Some(disk) = disk else {
continue;
};
let mut per_disk = info.clone();
per_disk.erasure.index = index;
candidates.push((disk, per_disk));
}
let owner = self
.attest_native_stream_owned(
store_id,
bucket,
key,
preparation,
incarnation,
candidates,
digest,
u64::try_from(size).map_err(Error::other)?,
fence,
Some(operation_owner),
)
.await?;
Ok((layout.write_quorum, owner.into_receipts()))
}
pub(crate) async fn prepare_native_bytes(
&self,
store_id: Uuid,
bucket: &str,
key: &str,
preparation: Uuid,
incarnation: Uuid,
body: Bytes,
fence: Arc<ObjectLockConfigSnapshot>,
saved_metadata: Vec<Option<Bytes>>,
operation_owner: Arc<rustfs_lock::NamespaceLockGuard>,
) -> Result<(usize, Vec<NativePreparationReceipt>)> {
let disks = self.get_disks_internal().await;
if disks.iter().flatten().any(|disk| matches!(disk.as_ref(), Disk::Remote(_))) {
return Err(Error::other("native remote preparation unsupported"));
}
let layout = resolve_write_layout(
&self.storage_class_config_snapshot(),
self.pool_index,
disks.len(),
self.default_parity_count,
None,
false,
)?;
let mut info = FileInfo::new(&format!("{bucket}/{key}"), layout.data_drives, layout.parity_drives);
info.name = key.to_owned();
// This is the actual private xl.meta version, not a claim about S3 readability.
info.version_id = Some(preparation);
info.data_dir = Some(preparation);
info.size = i64::try_from(body.len()).map_err(Error::other)?;
info.versioned = true;
// Mint one timestamp for new preparation; retries use the persisted writer value.
let mut modification_time = None;
for metadata in saved_metadata.iter().flatten() {
let saved = rustfs_filemeta::get_file_info(
metadata,
bucket,
key,
&preparation.to_string(),
FileInfoOpts {
data: false,
include_free_versions: false,
include_part_checksums: true,
},
)?;
let time = saved
.mod_time
.ok_or_else(|| Error::other("persisted native modification time missing"))?;
if modification_time.is_some_and(|old| old != time) {
return Err(Error::other("conflicting native modification times"));
}
modification_time = Some(time);
}
info.mod_time = Some(modification_time.unwrap_or_else(time::OffsetDateTime::now_utc));
info.add_object_part(1, String::new(), body.len(), None, info.size, None, None);
let erasure = Arc::new(Erasure::try_new_with_options(
info.erasure.data_blocks,
info.erasure.parity_blocks,
info.erasure.block_size,
false,
)?);
if body.len() > erasure.block_size || body.len() > 1024 * 1024 {
return Err(Error::other("native preparation requires bounded single-block inline data"));
}
if !self
.storage_class_config_snapshot()
.should_inline(erasure.shard_file_size(info.size), layout.data_drives, true)
{
return Err(Error::other("native preparation exceeds configured inline admission"));
}
let digest: [u8; 32] = Sha256::digest(&body).into();
let mut writers = std::iter::repeat_with(|| None).take(disks.len()).collect::<Vec<_>>();
let (_, size, shards, _) = erasure
.encode_with_shard_integrity(
Cursor::new(body),
&mut writers,
layout.write_quorum,
1,
IntegrityEncodeMode::Inline(usize::try_from(info.size).map_err(Error::other)?),
false,
)
.await?;
if i64::try_from(size).map_err(Error::other)? != info.size {
return Err(Error::other("native encoder length mismatch"));
}
let shards = shards.ok_or_else(|| Error::other("native inline encoder omitted shards"))?;
let (send, mut receive) = mpsc::unbounded_channel();
for (slot, disk) in disks.into_iter().enumerate() {
let Some(disk) = disk else {
continue;
};
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(Error::other("native remote preparation owner is not supported"));
};
let local = wrapper.get_disk();
let mut per_disk = info.clone();
let index = *info
.erasure
.distribution
.get(slot)
.ok_or_else(|| Error::other("native disk distribution missing"))?;
per_disk.erasure.index = index;
per_disk.data = Some(
shards
.get(
index
.checked_sub(1)
.ok_or_else(|| Error::other("invalid native shard index"))?,
)
.cloned()
.ok_or_else(|| Error::other("native encoder omitted assigned shard"))?,
);
per_disk.set_inline_data();
let metadata = if let Some(bytes) = saved_metadata.get(slot).and_then(Option::as_ref) {
let saved = rustfs_filemeta::get_file_info(
bytes,
bucket,
key,
&preparation.to_string(),
FileInfoOpts {
data: true,
include_free_versions: false,
include_part_checksums: true,
},
)?;
if saved.version_id != per_disk.version_id
|| saved.deleted
|| saved.mark_deleted
|| saved.uses_legacy_checksum != per_disk.uses_legacy_checksum
|| saved.mod_time != per_disk.mod_time
|| saved.size != per_disk.size
|| saved.erasure.distribution != per_disk.erasure.distribution
|| saved.erasure.index != index
|| saved.erasure.block_size != per_disk.erasure.block_size
|| saved.erasure.data_blocks != layout.data_drives
|| saved.erasure.parity_blocks != layout.parity_drives
|| saved.data != per_disk.data
{
return Err(Error::other("persisted native shard does not encode the request content"));
}
bytes.clone()
} else {
let mut encoded = FileMeta::new();
encoded.add_version(per_disk)?;
Bytes::from(encoded.marshal_msg()?)
};
let input = NativePrepareInput {
preparation,
store_id,
bucket: bucket.to_owned(),
bucket_incarnation: incarnation,
object_digest: digest,
key: key.to_owned(),
version: preparation,
metadata,
bucket_fence: fence.clone(),
};
let sender = send.clone();
let parent = operation_owner.clone();
// No per-disk lock is acquired here. NP01 owns its actual lock chain.
// Detached tasks retain their owners after quorum or caller cancellation;
// an unobserved reply leaves durable facts for probe/recovery, never rollback.
tokio::spawn(async move {
let result = match local.prepare_native_with_owner(input, parent).await {
Ok(receipt) => Ok(Some(receipt)),
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData => Err(Error::from(error)),
Err(_) => Ok(None),
};
let _ = sender.send(result);
});
}
drop(send);
let mut receipts = Vec::with_capacity(layout.write_quorum);
while let Some(result) = receive.recv().await {
if let Some(receipt) = result? {
if let Some(first) = receipts.first() {
if !same_identity(first, &receipt) {
return Err(Error::other("conflicting native preparation identities"));
}
}
if receipts
.iter()
.any(|r: &NativePreparationReceipt| r.disk_id() == receipt.disk_id())
{
return Err(Error::other("duplicate native physical disk"));
}
receipts.push(receipt);
if receipts.len() >= layout.write_quorum {
return Ok((layout.write_quorum, receipts));
}
}
}
Err(Error::ErasureWriteQuorum)
}
pub(crate) async fn recover_native_bytes(
&self,
store_id: Uuid,
bucket: &str,
key: &str,
preparation: Uuid,
incarnation: Uuid,
fence: Arc<ObjectLockConfigSnapshot>,
prepared_slots: Vec<bool>,
) -> Result<(usize, Vec<NativePreparationReceipt>, Vec<(Uuid, Bytes)>)> {
let disks = self.get_disks_internal().await;
let expected_set = self.native_expected_set_digest(store_id)?;
let layout = resolve_write_layout(
&self.storage_class_config_snapshot(),
self.pool_index,
disks.len(),
self.default_parity_count,
None,
false,
)?;
let mut receipts: Vec<NativePreparationReceipt> = Vec::new();
let mut verified_metadata = Vec::new();
let mut sources: Vec<Option<(crate::disk::DiskStore, FileInfo)>> =
std::iter::repeat_with(|| None).take(disks.len()).collect();
let mut erasure = None;
let mut modification_time = None;
let mut legacy_codec = None;
let mut block_size = None;
// Inspect every available disk: an identity conflict cannot be majority-voted away.
for (slot, disk) in disks.into_iter().enumerate() {
if !prepared_slots.get(slot).copied().unwrap_or(false) {
continue;
}
let disk = disk.ok_or_else(|| Error::other("prepared native disk unavailable"))?;
let (receipt, metadata) = match disk
.native_recover(preparation, store_id, bucket, key, incarnation, fence.clone())
.await
{
Ok(value) => value,
Err(error) => return Err(error.into()),
};
if receipt.set_digest() != &expected_set {
return Err(Error::other("native recovery receipt target physical set mismatch"));
}
if receipt.key() != key
|| receipt.preparation_id() != preparation
|| receipt.bucket() != bucket
|| receipt.bucket_incarnation() != incarnation
|| receipt.deployment_id() != store_id
{
return Err(Error::other("native recovery request identity mismatch"));
}
if let Some(first) = receipts.first() {
if !same_identity(first, &receipt) {
return Err(Error::other("conflicting native recovery identities"));
}
}
if receipts.iter().any(|r| r.disk_id() == receipt.disk_id()) {
return Err(Error::other("duplicate native recovery disk"));
}
verified_metadata.push((receipt.disk_id(), metadata.clone()));
let info = rustfs_filemeta::get_file_info(
&metadata,
bucket,
key,
&receipt.version_id().to_string(),
FileInfoOpts {
data: true,
include_free_versions: false,
include_part_checksums: true,
},
)?;
let time = info
.mod_time
.ok_or_else(|| Error::other("recovered native modification time missing"))?;
if modification_time.is_some_and(|old| old != time) {
return Err(Error::other("conflicting recovered native modification times"));
}
modification_time = Some(time);
if info.erasure.data_blocks != layout.data_drives || info.erasure.parity_blocks != layout.parity_drives {
return Err(Error::other("native recovery layout changed"));
}
if block_size.is_some_and(|old| old != info.erasure.block_size) {
return Err(Error::other("conflicting recovered native block sizes"));
}
block_size = Some(info.erasure.block_size);
if legacy_codec.is_some_and(|old| old != info.uses_legacy_checksum) {
return Err(Error::other("conflicting recovered native codecs"));
}
legacy_codec = Some(info.uses_legacy_checksum);
let selected = Erasure::try_new_with_options(
info.erasure.data_blocks,
info.erasure.parity_blocks,
info.erasure.block_size,
info.uses_legacy_checksum,
)?;
let index = info
.erasure
.index
.checked_sub(1)
.ok_or_else(|| Error::other("invalid recovered native index"))?;
let slot = sources
.get_mut(index)
.ok_or_else(|| Error::other("recovered native index outside set"))?;
if slot.is_some() {
return Err(Error::other("duplicate recovered native shard"));
}
if let Some((_, previous)) = sources.iter().flatten().next() {
if previous.parts != info.parts || previous.data_dir != info.data_dir {
return Err(Error::other("conflicting native recovered part manifest"));
}
}
sources[index] = Some((disk.clone(), info));
erasure = Some(selected);
receipts.push(receipt);
}
if receipts.len() < layout.write_quorum {
return Err(Error::ErasureWriteQuorum);
}
let first = receipts.first().ok_or(Error::ErasureWriteQuorum)?;
let length = usize::try_from(first.object_length()).map_err(Error::other)?;
let mut output = DigestSink {
digest: Sha256::new(),
length: 0,
};
let codec = erasure.ok_or(Error::ErasureWriteQuorum)?;
let parts = sources
.iter()
.flatten()
.next()
.ok_or(Error::ErasureWriteQuorum)?
.1
.parts
.clone();
let mut size = 0_usize;
for part in parts {
let mut readers = Vec::with_capacity(sources.len());
for source in &sources {
let Some((disk, info)) = source else {
readers.push(None);
continue;
};
let checksum = info.erasure.get_checksum_info(part.number).algorithm;
let checksum = if info.uses_legacy_checksum && checksum == rustfs_utils::HashAlgorithm::HighwayHash256S {
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum
};
let reader = if info.inline_data() {
crate::io_support::bitrot::ShardReader::InMemory(Cursor::new(
info.data.clone().ok_or_else(|| Error::other("native inline shard missing"))?,
))
} else {
use crate::disk::DiskAPI;
let data_dir = info.data_dir.ok_or_else(|| Error::other("native stream data dir missing"))?;
let shard_size = usize::try_from(codec.shard_file_size(i64::try_from(part.size).map_err(Error::other)?))
.map_err(Error::other)?;
let physical = if matches!(
checksum,
rustfs_utils::HashAlgorithm::HighwayHash256S | rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
) {
shard_size
.div_ceil(codec.shard_size())
.checked_mul(checksum.size())
.and_then(|hashes| shard_size.checked_add(hashes))
.ok_or_else(|| Error::other("native physical shard length overflow"))?
} else {
shard_size
};
crate::io_support::bitrot::ShardReader::Stream(
disk.read_file_stream(
crate::disk::RUSTFS_META_BUCKET,
&format!("native-preparations/{preparation}/{data_dir}/part.{}", part.number),
0,
physical,
)
.await?,
)
};
readers.push(Some(BitrotReader::new(reader, codec.shard_size(), checksum, false)));
}
let (decoded, failure) = codec.decode(&mut output, readers, 0, part.size, part.size).await;
if let Some(error) = failure {
return Err(error.into());
}
if decoded != part.size {
return Err(Error::other("native recovered part length mismatch"));
}
size = size
.checked_add(decoded)
.ok_or_else(|| Error::other("native decoded length overflow"))?;
}
if size != length || output.length != length || output.digest.finalize().as_slice() != first.object_digest() {
return Err(Error::other("native recovered content digest mismatch"));
}
Ok((layout.write_quorum, receipts, verified_metadata))
}
}
impl SetDisks {
pub(crate) async fn probe_native_candidates(
&self,
store_id: Uuid,
bucket: &str,
key: &str,
preparation: Uuid,
incarnation: Uuid,
fence: Arc<ObjectLockConfigSnapshot>,
match_object: bool,
) -> Vec<std::result::Result<NativePreparationProbe, DiskError>> {
let mut observations = Vec::new();
for disk in self.get_disks_internal().await {
observations.push(match disk {
Some(disk) => {
if match_object {
match disk.native_locate(store_id, preparation).await {
Ok(Some(record))
if record.bucket != bucket || record.key != key || record.incarnation != incarnation =>
{
observations.push(Ok(NativePreparationProbe::Absent));
continue;
}
Err(error) => {
observations.push(Err(error));
continue;
}
_ => {}
}
}
disk.native_probe(preparation, store_id, bucket, key, incarnation, fence.clone())
.await
}
None => Err(DiskError::InvalidPath),
});
}
observations
}
}
+5
View File
@@ -437,6 +437,11 @@ mod init;
pub(crate) mod init_format;
pub(crate) mod list_objects;
mod multipart;
pub mod native_prepare;
#[cfg(all(test, unix))]
mod native_prepare_multi_pool_tests;
#[cfg(all(test, unix))]
mod native_prepare_tests;
mod object;
#[cfg(feature = "test-util")]
pub use object::DeleteAfterObjectLockSnapshotBarrier;
+609
View File
@@ -0,0 +1,609 @@
//! Private native object preparation; a data quorum is not publication authority.
use super::ECStore;
use crate::bucket::utils::check_put_object_args;
use crate::disk::DiskAPI;
use crate::{
disk::local::native_prepare::{
NativePreparationDescription, NativePreparationProbe, NativePreparationReceipt, NativePrepareInput,
},
error::{Error, Result},
object_api::PutObjReader,
};
use bytes::Bytes;
use uuid::Uuid;
/// Identity supplied to a node-local native preparation operation.
///
/// `store_id` and `bucket_incarnation` are expectations to validate, not
/// authorization tokens. The receiving ECStore establishes its own fence.
#[derive(Clone, Debug)]
pub struct NativePreparationTarget {
pub store_id: Uuid,
pub disk_id: Uuid,
pub bucket: String,
pub key: String,
pub preparation: Uuid,
pub bucket_incarnation: Uuid,
}
/// Metadata and content identity for finalizing a shard already streamed to a disk.
pub struct NativePreparationFinalizeRequest {
pub target: NativePreparationTarget,
pub version: Uuid,
pub object_digest: [u8; 32],
pub metadata: Bytes,
}
/// Verified local recovery data returned by a target owner.
pub struct NativePreparationRecovery {
pub receipt: NativePreparationReceipt,
pub metadata: Bytes,
}
pub struct NativeQuorumPreparation {
receipts: Vec<NativePreparationReceipt>,
}
impl NativeQuorumPreparation {
pub(crate) fn into_receipts(self) -> Vec<NativePreparationReceipt> {
self.receipts
}
pub(crate) fn receipts(&self) -> &[NativePreparationReceipt] {
&self.receipts
}
pub fn preparation_id(&self) -> Uuid {
self.receipts[0].preparation_id()
}
pub fn version_id(&self) -> Uuid {
self.receipts[0].version_id()
}
pub fn object_digest(&self) -> [u8; 32] {
*self.receipts[0].object_digest()
}
pub fn object_length(&self) -> u64 {
self.receipts[0].object_length()
}
pub fn receipt_count(&self) -> usize {
self.receipts().len()
}
pub fn disk_ids(&self) -> impl Iterator<Item = Uuid> + '_ {
self.receipts.iter().map(NativePreparationReceipt::disk_id)
}
}
#[derive(Clone, PartialEq, Eq)]
struct Identity {
version: Uuid,
digest: [u8; 32],
length: u64,
set: [u8; 32],
distribution: [u8; 32],
}
struct Observation {
set: std::sync::Arc<crate::set_disk::SetDisks>,
identity: Option<Identity>,
metadata: Vec<Option<bytes::Bytes>>,
prepared: Vec<bool>,
unknown: bool,
}
impl NativeQuorumPreparation {
pub(crate) fn from_receipts(quorum: usize, receipts: Vec<NativePreparationReceipt>) -> Result<Self> {
let first = receipts.first().ok_or_else(|| Error::other("empty native quorum"))?;
let distinct: std::collections::HashSet<_> = receipts.iter().map(NativePreparationReceipt::disk_id).collect();
if receipts.len() < quorum
|| distinct.len() != receipts.len()
|| receipts.iter().any(|r| !crate::set_disk::same_identity(first, r))
{
return Err(Error::other("inconsistent native quorum"));
}
Ok(Self { receipts })
}
}
impl ECStore {
/// Locate a durable preparation on the exact local disk named by the caller.
/// The description is local locator data and is not a preparation receipt.
#[cfg(unix)]
pub async fn locate_native_preparation_on_disk(
&self,
disk_id: Uuid,
store_id: Uuid,
preparation: Uuid,
) -> Result<Option<NativePreparationDescription>> {
if store_id != self.id || store_id.is_nil() || disk_id.is_nil() || preparation.is_nil() {
return Err(Error::other("native preparation locator identity mismatch"));
}
let disk = self.resolve_native_local_disk(disk_id).await?;
disk.locate_native_preparation(store_id, preparation)
.await
.map_err(Into::into)
}
#[cfg(not(unix))]
pub async fn locate_native_preparation_on_disk(
&self,
_disk_id: Uuid,
_store_id: Uuid,
_preparation: Uuid,
) -> Result<Option<NativePreparationDescription>> {
Err(Error::UnsupportedDisk)
}
async fn validate_native_target(
&self,
target: &NativePreparationTarget,
) -> Result<std::sync::Arc<crate::object_api::ObjectLockConfigSnapshot>> {
if target.store_id != self.id
|| self.id.is_nil()
|| target.disk_id.is_nil()
|| target.preparation.is_nil()
|| target.bucket_incarnation.is_nil()
{
return Err(Error::other("native preparation target identity mismatch"));
}
check_put_object_args(&target.bucket, &target.key)?;
let fence = self.object_lock_config_snapshot(&target.bucket).await?;
let incarnation = self.bucket_incarnation_id_from_disk(&target.bucket).await?;
if incarnation != target.bucket_incarnation || !fence.is_valid_for_destructive_put(self.id, &target.bucket, incarnation) {
return Err(Error::other("native preparation target bucket fence mismatch"));
}
Ok(fence)
}
async fn resolve_native_local_disk(&self, disk_id: Uuid) -> Result<std::sync::Arc<crate::disk::local::LocalDisk>> {
use crate::disk::{Disk, DiskAPI};
let mut matches = Vec::new();
let mut lookup_error = None;
for pool in &self.pools {
for set in &pool.disk_set {
let disks = set.disks.read().await.clone();
for disk in disks.into_iter().flatten() {
let Disk::Local(local) = disk.as_ref() else {
continue;
};
let local_disk = local.get_disk();
match local_disk.get_disk_id().await {
Ok(Some(id)) if id == disk_id => matches.push(local_disk),
Ok(_) => {}
Err(error) => {
lookup_error.get_or_insert(error);
}
}
}
}
}
if matches.len() != 1 {
if matches.is_empty()
&& let Some(error) = lookup_error
{
return Err(error.into());
}
return Err(Error::other("native preparation target disk is not uniquely local"));
}
matches
.pop()
.ok_or_else(|| Error::other("native preparation target disk disappeared"))
}
/// Create private stream staging on the exact local disk named by the caller.
/// The caller holds the distributed object lock across all disk RPCs; this owner must not
/// reacquire that non-reentrant lock. Store and bucket identities are checked against fresh
/// local state, while disk operations take their own physical mutation locks.
pub async fn begin_native_preparation_on_disk(&self, target: &NativePreparationTarget) -> Result<()> {
let fence = self.validate_native_target(target).await?;
let disk = self.resolve_native_local_disk(target.disk_id).await?;
disk.begin_native_stream(target.preparation, self.id, &target.bucket, target.bucket_incarnation, fence)
.await?;
Ok(())
}
/// Verify and durably attest the stream already written to the named local disk.
/// This does not select placement or encode the object.
pub async fn finalize_native_preparation_on_disk(
&self,
request: NativePreparationFinalizeRequest,
) -> Result<NativePreparationReceipt> {
let target = &request.target;
let fence = self.validate_native_target(target).await?;
let disk = self.resolve_native_local_disk(target.disk_id).await?;
disk.prepare_native(NativePrepareInput {
preparation: target.preparation,
store_id: self.id,
bucket: target.bucket.clone(),
bucket_incarnation: target.bucket_incarnation,
object_digest: request.object_digest,
key: target.key.clone(),
version: request.version,
metadata: request.metadata,
bucket_fence: fence,
})
.await
.map_err(Into::into)
}
/// Inspect the named local disk for a preparation without trusting a remote fence.
pub async fn probe_native_preparation_on_disk(&self, target: &NativePreparationTarget) -> Result<NativePreparationProbe> {
let fence = self.validate_native_target(target).await?;
let disk = self.resolve_native_local_disk(target.disk_id).await?;
disk.probe_native(target.preparation, self.id, &target.bucket, &target.key, target.bucket_incarnation, fence)
.await
.map_err(Into::into)
}
/// Recover and revalidate durable preparation state from the named local disk.
pub async fn recover_native_preparation_on_disk(
&self,
target: &NativePreparationTarget,
) -> Result<NativePreparationRecovery> {
let fence = self.validate_native_target(target).await?;
let disk = self.resolve_native_local_disk(target.disk_id).await?;
let (receipt, metadata) = disk
.recover_native_with_metadata(target.preparation, self.id, &target.bucket, target.bucket_incarnation, fence)
.await?;
Ok(NativePreparationRecovery { receipt, metadata })
}
async fn native_observations(
&self,
bucket: &str,
key: &str,
preparation: Uuid,
incarnation: Uuid,
fence: std::sync::Arc<crate::object_api::ObjectLockConfigSnapshot>,
) -> Result<Vec<Observation>> {
self.native_observations_scan(bucket, key, preparation, incarnation, fence, false)
.await
}
async fn native_observations_scan(
&self,
bucket: &str,
key: &str,
preparation: Uuid,
incarnation: Uuid,
fence: std::sync::Arc<crate::object_api::ObjectLockConfigSnapshot>,
all_sets: bool,
) -> Result<Vec<Observation>> {
let mut pools = Vec::new();
for pool in &self.pools {
let sets = if all_sets {
pool.disk_set.clone()
} else {
vec![pool.get_disks_by_key(key)]
};
for set in sets {
let candidates = set
.probe_native_candidates(self.id, bucket, key, preparation, incarnation, fence.clone(), all_sets)
.await;
let mut observed = Observation {
set,
identity: None,
metadata: Vec::new(),
prepared: Vec::new(),
unknown: false,
};
for candidate in candidates {
let mut bytes = None;
let mut prepared = false;
match candidate {
Ok(crate::disk::local::native_prepare::NativePreparationProbe::Absent) => {}
Ok(crate::disk::local::native_prepare::NativePreparationProbe::Existing {
version,
prepared: ready,
object_digest,
object_length,
set_digest,
distribution_digest,
metadata,
}) => {
let identity = Identity {
version,
digest: object_digest,
length: object_length,
set: set_digest,
distribution: distribution_digest,
};
if observed.identity.as_ref().is_some_and(|old| old != &identity) {
return Err(Error::other("conflicting native preparation identity"));
}
observed.identity = Some(identity);
bytes = metadata;
prepared = ready;
}
Err(crate::disk::error::Error::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData => {
return Err(Error::from(error));
}
Err(_) => observed.unknown = true,
}
observed.metadata.push(bytes);
observed.prepared.push(prepared);
}
pools.push(observed);
}
}
Ok(pools)
}
fn existing_native_pool(observations: &[Observation]) -> Result<Option<usize>> {
let existing: Vec<_> = observations
.iter()
.enumerate()
.filter(|(_, p)| p.identity.is_some())
.map(|(i, _)| i)
.collect();
if existing.len() > 1 {
return Err(Error::other("native preparation exists in multiple pools"));
}
let selected = existing.first().copied();
if observations.len() > 1 && observations.iter().enumerate().any(|(i, p)| p.unknown && Some(i) != selected) {
return Err(Error::other("native preparation pool attribution is unknown"));
}
Ok(selected)
}
pub async fn prepare_native_object(
&self,
bucket: &str,
key: &str,
preparation: Uuid,
data: &mut PutObjReader,
) -> Result<NativeQuorumPreparation> {
use sha2::{Digest, Sha256};
use tokio::io::AsyncReadExt;
check_put_object_args(bucket, key)?;
if preparation.is_nil() || self.pools.is_empty() || data.size() < -1 {
return Err(Error::other("unsupported native preparation"));
}
let mut remote = false;
for pool in &self.pools {
if pool
.get_disks_by_key(key)
.disks
.read()
.await
.iter()
.flatten()
.any(|disk| !disk.is_local())
{
remote = true;
break;
}
}
if remote || data.size() < 0 || data.size() > 1024 * 1024 {
let fence = self.object_lock_config_snapshot(bucket).await?;
let incarnation = self.bucket_incarnation_id(bucket).await?;
let owner = std::sync::Arc::new(
self.handle_new_ns_lock(bucket, key)
.await?
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await?,
);
let observations = self
.native_observations_scan(bucket, key, preparation, incarnation, fence.clone(), true)
.await?;
if let Some(selected) = Self::existing_native_pool(&observations)? {
let identity = observations[selected]
.identity
.as_ref()
.ok_or_else(|| Error::other("missing native identity"))?;
let mut hash = Sha256::new();
let mut length = 0_u64;
let mut buffer = vec![0_u8; 64 * 1024];
loop {
let size = data.stream.read(&mut buffer).await?;
if size == 0 {
break;
}
length = length
.checked_add(u64::try_from(size).map_err(Error::other)?)
.ok_or_else(|| Error::other("native retry length overflow"))?;
hash.update(&buffer[..size]);
}
if hash.finalize().as_slice() != identity.digest || length != identity.length {
return Err(Error::other("native streaming retry content conflict"));
}
drop(owner);
return self.recover_native_object(bucket, key, preparation).await;
}
let selected = if self.pools.len() == 1 {
0
} else {
self.get_pool_idx_no_lock(bucket, key, data.size().max(0)).await?
};
let set = self.pools[selected].get_disks_by_key(key);
let (quorum, receipts) =
Box::pin(set.prepare_native_stream(self.id, bucket, key, preparation, incarnation, data, fence, owner)).await?;
return NativeQuorumPreparation::from_receipts(quorum, receipts);
}
let length = data.size();
let mut body = Vec::new();
(&mut data.stream).take(1024 * 1024 + 1).read_to_end(&mut body).await?;
if body.len() as i64 != length {
return Err(Error::other("native reader length mismatch"));
}
let digest: [u8; 32] = Sha256::digest(&body).into();
let fence = self.object_lock_config_snapshot(bucket).await?;
let incarnation = self.bucket_incarnation_id(bucket).await?;
let owner = std::sync::Arc::new(
self.handle_new_ns_lock(bucket, key)
.await?
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await?,
);
let mut observations = self
.native_observations(bucket, key, preparation, incarnation, fence.clone())
.await?;
let selected = match Self::existing_native_pool(&observations)? {
Some(index) => index,
None if observations.len() == 1 => 0,
None => self.get_pool_idx_no_lock(bucket, key, length).await?,
};
let observed = observations.remove(selected);
if observed
.identity
.as_ref()
.is_some_and(|identity| identity.digest != digest || identity.length != body.len() as u64)
{
return Err(Error::other("native preparation content conflict"));
}
let (quorum, receipts) = self.pools[selected]
.get_disks_by_key(key)
.prepare_native_bytes(
self.id,
bucket,
key,
preparation,
incarnation,
body.into(),
fence,
observed.metadata,
owner,
)
.await?;
NativeQuorumPreparation::from_receipts(quorum, receipts)
}
pub async fn recover_native_object(&self, bucket: &str, key: &str, preparation: Uuid) -> Result<NativeQuorumPreparation> {
check_put_object_args(bucket, key)?;
if preparation.is_nil() || self.pools.is_empty() {
return Err(Error::other("invalid native preparation"));
}
let _owner = self
.handle_new_ns_lock(bucket, key)
.await?
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await?;
self.recover_native_object_unlocked(bucket, key, preparation)
.await
.map(|(owner, _)| owner)
}
/// Caller owns the distributed namespace lock; each disk still acquires its actual physical recovery owners.
pub(crate) async fn recover_native_object_unlocked(
&self,
bucket: &str,
key: &str,
preparation: Uuid,
) -> Result<(NativeQuorumPreparation, Vec<(Uuid, bytes::Bytes)>)> {
check_put_object_args(bucket, key)?;
if preparation.is_nil() || self.pools.is_empty() {
return Err(Error::other("invalid native preparation"));
}
let fence = self.object_lock_config_snapshot(bucket).await?;
let incarnation = self.bucket_incarnation_id(bucket).await?;
let observations = self
.native_observations_scan(bucket, key, preparation, incarnation, fence.clone(), true)
.await?;
let selected = Self::existing_native_pool(&observations)?.ok_or_else(|| Error::other("native preparation absent"))?;
let (quorum, receipts, metadata) = observations[selected]
.set
.recover_native_bytes(
self.id,
bucket,
key,
preparation,
incarnation,
fence,
observations[selected].prepared.clone(),
)
.await?;
Ok((NativeQuorumPreparation::from_receipts(quorum, receipts)?, metadata))
}
}
#[cfg(all(test, unix))]
mod target_owner_tests {
use super::*;
use crate::{
bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks},
disk::DiskAPI,
object_api::PutObjReader,
storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions},
};
use uuid::Uuid;
#[tokio::test]
#[serial_test::serial]
async fn target_owner_apis_use_fresh_local_identity_and_exact_disk() {
let (dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "native-target-owner";
let key = "object.bin";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create target bucket");
let preparation = Uuid::new_v4();
let body = b"target-owner facade identity";
let prepared = store
.prepare_native_object(bucket, key, preparation, &mut PutObjReader::from_vec(body.to_vec()))
.await
.expect("prepare native object");
let disk = store.pools[0].get_disks_by_key(key).disks.read().await[0]
.clone()
.expect("first local disk");
let disk_id = disk.get_disk_id().await.expect("disk identity").expect("formatted disk");
assert!(prepared.disk_ids().any(|id| id == disk_id));
let target = NativePreparationTarget {
store_id: store.id,
disk_id,
bucket: bucket.into(),
key: key.into(),
preparation,
bucket_incarnation: store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("durable bucket incarnation"),
};
let metadata_path = dirs[0]
.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string())
.join("xl.meta");
let metadata = Bytes::from(std::fs::read(metadata_path).expect("persisted shard metadata"));
let unrelated_format = dirs[1].path().join(".rustfs.sys/format.json");
let saved_unrelated_format = std::fs::read(&unrelated_format).expect("unrelated disk format");
std::fs::write(&unrelated_format, b"invalid format").expect("damage unrelated disk format");
let caller_owner = store
.handle_new_ns_lock(bucket, key)
.await
.expect("caller namespace lock")
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await
.expect("caller owns namespace lock");
let receipt = store
.finalize_native_preparation_on_disk(NativePreparationFinalizeRequest {
target: target.clone(),
version: preparation,
object_digest: prepared.object_digest(),
metadata: metadata.clone(),
})
.await
.expect("target-owner idempotent finalize");
assert_eq!(receipt.disk_id(), disk_id);
assert_eq!(receipt.object_digest(), &prepared.object_digest());
assert!(matches!(
tokio::time::timeout(
std::time::Duration::from_secs(2),
store.probe_native_preparation_on_disk(&target),
)
.await
.expect("target-owner must not reacquire the caller's non-reentrant lock")
.expect("target-owner probe"),
NativePreparationProbe::Existing { prepared: true, metadata: Some(found), .. } if found == metadata
));
let recovered = store
.recover_native_preparation_on_disk(&target)
.await
.expect("target-owner recovery");
assert_eq!(recovered.receipt.disk_id(), disk_id);
assert_eq!(recovered.metadata, metadata);
let mut wrong_incarnation = target.clone();
wrong_incarnation.bucket_incarnation = Uuid::new_v4();
assert!(store.begin_native_preparation_on_disk(&wrong_incarnation).await.is_err());
let mut wrong_disk = target;
wrong_disk.disk_id = Uuid::new_v4();
assert!(store.begin_native_preparation_on_disk(&wrong_disk).await.is_err());
drop(caller_owner);
std::fs::write(unrelated_format, saved_unrelated_format).expect("restore unrelated disk format");
}
}
@@ -0,0 +1,278 @@
//! Real private preparation placement across two physical local pools.
use super::ECStore;
use crate::{
bucket::metadata_sys::init_bucket_metadata_sys,
disk::endpoint::Endpoint,
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
object_api::PutObjReader,
runtime::instance::InstanceContext,
storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions},
store::init_local_disks_with_instance_ctx,
};
use sha2::{Digest, Sha256};
use std::sync::Arc;
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const BUCKET: &str = "native-multi-pool-preparation";
const KEY: &str = "private/placement.bin";
const CONTENT: &[u8] = b"actual two-pool reader bytes\0remain bound to durable placement";
const DRIVES: usize = 4;
async fn reopen(dirs: &[TempDir]) -> Arc<ECStore> {
assert_eq!(dirs.len(), 2 * DRIVES);
let pools = (0..2)
.map(|pool| {
let endpoints = (0..DRIVES)
.map(|slot| {
let mut endpoint = Endpoint::try_from(dirs[pool * DRIVES + slot].path().to_str().expect("local path"))
.expect("actual local endpoint");
endpoint.set_pool_index(pool);
endpoint.set_set_index(0);
endpoint.set_disk_index(slot);
endpoint
})
.collect::<Vec<_>>();
PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: DRIVES,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("native-multi-pool-{pool}"),
platform: "test".into(),
}
})
.collect();
let pools = EndpointServerPools(pools);
let context = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&context, pools.clone())
.await
.expect("initialize real formats for both pools");
let store =
ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("local address"), pools, CancellationToken::new(), context)
.await
.expect("open actual two-pool store");
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
store
}
fn private_path(dir: &TempDir, preparation: Uuid) -> std::path::PathBuf {
dir.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string())
}
async fn create_bucket(store: &Arc<ECStore>) {
let store = store.clone();
tokio::spawn(async move { store.make_bucket(BUCKET, &MakeBucketOptions::default()).await })
.await
.expect("join actual bucket creation")
.expect("create actual bucket");
}
fn assert_only_pool(dirs: &[TempDir], preparation: Uuid, pool: usize) {
let other = 1 - pool;
for dir in &dirs[other * DRIVES..(other + 1) * DRIVES] {
assert!(
!private_path(dir, preparation).exists(),
"NP02_POOL_PLACEMENT_BOUND: retry must not create private facts in another pool"
);
}
}
async fn await_pending_owners(store: &ECStore) {
// The preparation's parent owner outlives its early Wdata return. Acquire the
// same real object lock before changing files in this test's fixture.
let lock = store.handle_new_ns_lock(BUCKET, KEY).await.expect("actual object lock");
let owner = lock
.get_write_lock(std::time::Duration::from_secs(30))
.await
.expect("pending disk owners finish before fixture mutation");
drop(owner);
}
#[tokio::test]
#[serial_test::serial]
async fn native_object_prepare_multi_pool_restart_pending_and_unknown_preserve_placement() {
let dirs = (0..2 * DRIVES)
.map(|_| tempfile::tempdir().expect("physical local disk"))
.collect::<Vec<_>>();
let store = reopen(&dirs).await;
create_bucket(&store).await;
let preparation = Uuid::new_v4();
let prepared = store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await
.expect("NP02_POOL_PLACEMENT_BOUND: two-pool native preparation is supported");
let version = prepared.version_id();
assert_eq!(prepared.object_digest(), <[u8; 32]>::from(Sha256::digest(CONTENT)));
let present = (0..2)
.filter(|pool| {
dirs[pool * DRIVES..(pool + 1) * DRIVES]
.iter()
.any(|dir| private_path(dir, preparation).exists())
})
.collect::<Vec<_>>();
assert_eq!(present.len(), 1, "NP02_POOL_PLACEMENT_BOUND: one actual durable pool");
let pool = present[0];
assert_only_pool(&dirs, preparation, pool);
drop(prepared);
await_pending_owners(&store).await;
// Recovery has to rediscover private placement from physical files in a fresh context.
let reopened = reopen(&dirs).await;
let recovered = reopened
.recover_native_object(BUCKET, KEY, preparation)
.await
.expect("NP02_POOL_PLACEMENT_BOUND: restart recovers original private pool");
assert_eq!(recovered.version_id(), version, "NP02_POOL_PLACEMENT_BOUND");
assert_eq!(recovered.object_digest(), <[u8; 32]>::from(Sha256::digest(CONTENT)));
drop(recovered);
await_pending_owners(&reopened).await;
let marker = dirs[pool * DRIVES..(pool + 1) * DRIVES]
.iter()
.map(|dir| private_path(dir, preparation).join("PREPARED"))
.find(|path| path.exists())
.expect("actual persisted acknowledgment");
let original_marker = std::fs::read(&marker).expect("retain original marker bytes");
std::fs::write(&marker, b"corrupt existing acknowledgment").expect("corrupt own actual marker");
let corrupt = reopened
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await;
assert!(
corrupt.is_err(),
"NP02_POOL_PLACEMENT_BOUND: a corrupted existing marker is not absent placement"
);
assert_only_pool(&dirs, preparation, pool);
await_pending_owners(&reopened).await;
std::fs::write(&marker, original_marker).expect("restore original actual marker");
// Keep durable intents while removing acknowledgments and xl.meta.
// Retry must restore each writer's original raw bytes in the same pool.
let mut original_metadata = Vec::new();
for dir in &dirs[pool * DRIVES..(pool + 1) * DRIVES] {
let marker = private_path(dir, preparation).join("PREPARED");
if marker.exists() {
std::fs::remove_file(marker).expect("remove own fixture acknowledgment");
}
let metadata = private_path(dir, preparation).join("xl.meta");
if metadata.exists() {
let bytes = std::fs::read(&metadata).expect("retain original raw metadata");
std::fs::remove_file(&metadata).expect("leave only original durable intent");
original_metadata.push((metadata, bytes));
}
}
let repaired = reopened
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await
.expect("NP02_POOL_PLACEMENT_BOUND: pending intents repair in their existing pool");
assert_eq!(repaired.version_id(), version, "NP02_POOL_PLACEMENT_BOUND");
drop(repaired);
await_pending_owners(&reopened).await;
assert!(
original_metadata.len() >= 3,
"NP02_POOL_PLACEMENT_BOUND: retain actual write quorum metadata"
);
for (path, bytes) in original_metadata {
assert_eq!(
std::fs::read(path).expect("repaired actual metadata file"),
bytes,
"NP02_POOL_PLACEMENT_BOUND: top-level retry restores original raw bytes from intent"
);
}
assert_only_pool(&dirs, preparation, pool);
// Preserve each original directory while making actual filesystem inspection fail.
// Public object absence must not turn this unknown private outcome into a new placement.
for dir in &dirs[pool * DRIVES..(pool + 1) * DRIVES] {
let root = dir.path().join(".rustfs.sys/native-preparations");
std::fs::rename(&root, dir.path().join(".rustfs.sys/native-preparations-preserved"))
.expect("retain original private facts");
std::fs::write(root, b"actual namespace obstruction").expect("install real I/O failure");
}
let retry = reopened
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await;
assert!(
retry.is_err(),
"NP02_POOL_PLACEMENT_BOUND: unknown original pool cannot migrate to healthy pool"
);
assert_only_pool(&dirs, preparation, pool);
}
#[tokio::test]
#[serial_test::serial]
async fn native_object_prepare_coherent_replay_rejects_changed_block_size() {
let (dirs, store) = Box::pin(crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks()).await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
create_bucket(&store).await;
let preparation = Uuid::new_v4();
let prepared = store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await
.expect("original real native preparation");
drop(prepared);
await_pending_owners(&store).await;
let private = private_path(&dirs[0], preparation);
let original = std::fs::read(private.join("xl.meta")).expect("original actual writer metadata");
let mut info = rustfs_filemeta::get_file_info(
&original,
BUCKET,
KEY,
&preparation.to_string(),
rustfs_filemeta::FileInfoOpts {
data: true,
include_free_versions: false,
include_part_checksums: true,
},
)
.expect("strictly decode real writer metadata");
let original_block_size = info.erasure.block_size;
info.erasure.block_size /= 2;
assert!(info.erasure.block_size > CONTENT.len());
assert_ne!(info.erasure.block_size, original_block_size);
let mut meta = rustfs_filemeta::FileMeta::new();
meta.add_version(info).expect("real metadata writer for changed geometry");
let changed = meta.marshal_msg().expect("encode actual changed raw metadata");
let mut encoded = std::fs::read_to_string(private.join("intent.json")).expect("actual canonical durable intent");
let original_intent: serde_json::Value = serde_json::from_str(&encoded).expect("decode actual intent");
// Preserve the original canonical field order. Change only the three raw-byte
// bindings, so this is a coherent candidate rather than a checksum mismatch.
for (field, value) in [
("metadata_digest", serde_json::json!(Sha256::digest(&changed).to_vec())),
("metadata_length", serde_json::json!(changed.len())),
("recovery_metadata", serde_json::json!(base64_simd::STANDARD.encode_to_string(&changed))),
] {
let old = format!("\"{field}\":{}", serde_json::to_string(&original_intent[field]).expect("original field"));
let new = format!("\"{field}\":{}", serde_json::to_string(&value).expect("changed field"));
assert_eq!(encoded.matches(&old).count(), 1, "only one original canonical binding");
encoded = encoded.replacen(&old, &new, 1);
}
std::fs::write(private.join("xl.meta"), &changed).expect("replace own actual shard metadata");
std::fs::write(private.join("intent.json"), encoded.as_bytes()).expect("retain coherent actual intent");
std::fs::write(private.join("PREPARED"), Sha256::digest(encoded.as_bytes())).expect("coherent existing marker");
let set = store.pools[0].get_disks_by_key(KEY);
let disk = set.disks.read().await[0].clone().expect("actual first physical disk");
let crate::disk::Disk::Local(local) = disk.as_ref() else {
panic!("local fixture disk");
};
let fence = store
.object_lock_config_snapshot(BUCKET)
.await
.expect("fresh real lifecycle fence");
let incarnation = store.bucket_incarnation_id(BUCKET).await.expect("actual bucket incarnation");
local
.get_disk()
.recover_native(preparation, store.id, BUCKET, incarnation, fence)
.await
.expect("NP02_GEOMETRY_IDENTITY_BOUND: changed short shard remains a valid real NP01 candidate");
assert!(
store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await
.is_err(),
"NP02_GEOMETRY_IDENTITY_BOUND: top-level retry rejects a different actual block geometry"
);
await_pending_owners(&store).await;
assert!(
store.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
"NP02_GEOMETRY_IDENTITY_BOUND: top-level recovery rejects mixed actual block geometry"
);
}
@@ -0,0 +1,981 @@
use super::ECStore;
use crate::{
bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks},
disk::endpoint::Endpoint,
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
object_api::{ObjectOptions, PutObjReader},
runtime::instance::InstanceContext,
storage_api_contracts::{
bucket::{BucketOperations as _, MakeBucketOptions},
object::ObjectOperations as _,
},
store::init_local_disks_with_instance_ctx,
};
use sha2::{Digest, Sha256};
use std::{collections::HashSet, sync::Arc};
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const BUCKET: &str = "native-quorum-preparation";
const KEY: &str = "nested/object.bin";
const CONTENT: &[u8] = b"native preparation content\0with an immutable identity";
async fn fixture() -> (Vec<TempDir>, Arc<ECStore>) {
let (dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
store
.make_bucket(BUCKET, &MakeBucketOptions::default())
.await
.expect("create actual four-disk bucket");
(dirs, store)
}
async fn multi_pool_fixture() -> (Vec<TempDir>, Arc<ECStore>) {
let dirs = (0..8)
.map(|_| tempfile::tempdir().expect("real local disk directory"))
.collect::<Vec<_>>();
let pools = EndpointServerPools(
(0..2)
.map(|pool_index| {
let endpoints = (0..4)
.map(|disk_index| {
let dir = &dirs[pool_index * 4 + disk_index];
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoint
})
.collect::<Vec<_>>();
PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "native-multi-pool-test".into(),
platform: "test".into(),
}
})
.collect(),
);
let instance = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance, pools.clone())
.await
.expect("initialize both physical pools");
let store =
ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), pools, CancellationToken::new(), instance)
.await
.expect("initialize multi-pool store");
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
store
.make_bucket(BUCKET, &MakeBucketOptions::default())
.await
.expect("create bucket across both pools");
(dirs, store)
}
async fn prepare_stream_in_pool(store: &Arc<ECStore>, pool_index: usize, preparation: Uuid, body: Vec<u8>) {
let incarnation = store.bucket_incarnation_id(BUCKET).await.expect("bucket incarnation");
let fence = store.object_lock_config_snapshot(BUCKET).await.expect("bucket fence");
let owner = Arc::new(
store
.handle_new_ns_lock(BUCKET, KEY)
.await
.expect("object namespace lock")
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await
.expect("object write lock"),
);
let mut reader = PutObjReader::from_vec(body);
let (quorum, receipts) = store.pools[pool_index]
.get_disks_by_key(KEY)
.prepare_native_stream(store.id, BUCKET, KEY, preparation, incarnation, &mut reader, fence, owner)
.await
.expect("prepare on the specified physical pool");
assert!(receipts.len() >= quorum, "NATIVE_CROSS_POOL_PREPARE_REQUIRES_WRITE_QUORUM");
}
async fn assert_not_published(store: &ECStore) {
let result = store.get_object_info(BUCKET, KEY, &ObjectOptions::default()).await;
assert!(
matches!(result, Err(ref error) if crate::error::is_err_object_not_found(error)),
"NP02_PREPARATION_NOT_AUTHORITY: native preparation must remain invisible to ordinary object HEAD"
);
}
fn fail_preparation_disk(dir: &TempDir) {
// Keep the physical disk and its format present; only native preparation I/O fails.
std::fs::write(dir.path().join(".rustfs.sys/native-preparations"), b"not a directory")
.expect("install real per-disk namespace obstruction");
}
async fn reopen(dirs: &[TempDir]) -> Arc<ECStore> {
let endpoints = dirs
.iter()
.enumerate()
.map(|(index, dir)| {
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("existing local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(index / 4);
endpoint.set_disk_index(index % 4);
endpoint
})
.collect::<Vec<_>>();
let pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: dirs.len() / 4,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "native-quorum-reopen".into(),
platform: "test".into(),
}]);
let context = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&context, pools.clone())
.await
.expect("reopen existing disk formats");
let store =
ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("local address"), pools, CancellationToken::new(), context)
.await
.expect("restart store over the same physical disks");
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
store
}
#[tokio::test]
#[serial_test::serial]
async fn native_object_prepare_one_disk_failure_retains_write_quorum() {
let (dirs, store) = fixture().await;
fail_preparation_disk(&dirs[0]);
let preparation = Uuid::new_v4();
let owner = store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await;
assert!(
owner.is_ok(),
"NP02_WDATA_BOUND: N4/W3 must tolerate one actual disk I/O failure: {:?}",
owner.as_ref().err()
);
let owner = owner.expect("checked preparation result");
assert_eq!(owner.preparation_id(), preparation, "NP02_WDATA_BOUND");
assert_eq!(
owner.object_digest(),
<[u8; 32]>::from(Sha256::digest(CONTENT)),
"NP02_OPERATION_IDENTITY_BOUND: digest computed from actual reader bytes"
);
assert_eq!(owner.receipt_count(), 3, "NP02_WDATA_BOUND: three real durable receipts");
assert_eq!(
owner.disk_ids().collect::<HashSet<_>>().len(),
3,
"NP02_WDATA_BOUND: distinct physical disks"
);
assert_eq!(owner.object_length(), u64::try_from(CONTENT.len()).expect("small content"));
assert!(!owner.version_id().is_nil(), "NP02_WDATA_BOUND: real immutable version");
drop(owner);
assert_not_published(&store).await;
}
#[tokio::test]
#[serial_test::serial]
async fn native_object_prepare_below_write_quorum_has_no_authority() {
let (dirs, store) = fixture().await;
fail_preparation_disk(&dirs[0]);
fail_preparation_disk(&dirs[1]);
let preparation = Uuid::new_v4();
let prepared = store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await;
assert!(
prepared.is_err(),
"NP02_PREPARATION_NOT_AUTHORITY: W-1 must not construct preparation authority"
);
drop(prepared);
let recovered = store.recover_native_object(BUCKET, KEY, preparation).await;
assert!(
recovered.is_err(),
"NP02_PREPARATION_NOT_AUTHORITY: partial durable facts are not quorum authority"
);
drop(recovered);
assert_not_published(&store).await;
// Recovery after a new store context must apply the same threshold to those facts.
drop(store);
let restarted = reopen(&dirs).await;
assert!(
restarted.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
"NP02_PREPARATION_NOT_AUTHORITY: restart cannot promote W-1 facts"
);
}
#[tokio::test]
#[serial_test::serial]
async fn native_object_prepare_restart_preserves_operation_and_content_identity() {
let (dirs, store) = fixture().await;
let preparation = Uuid::new_v4();
let owner = store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await;
assert!(
owner.is_ok(),
"NP02_OPERATION_IDENTITY_BOUND: prepare actual native object: {:?}",
owner.as_ref().err()
);
let owner = owner.expect("checked preparation result");
let expected = (owner.preparation_id(), owner.version_id(), owner.object_digest(), owner.object_length());
assert!(owner.receipt_count() >= 3, "NP02_WDATA_BOUND: durable write quorum");
assert_eq!(
owner.disk_ids().collect::<HashSet<_>>().len(),
owner.receipt_count(),
"NP02_WDATA_BOUND: independent disk facts"
);
drop(owner);
drop(store);
let restarted = reopen(&dirs).await;
let recovered = restarted.recover_native_object(BUCKET, KEY, preparation).await;
assert!(
recovered.is_ok(),
"NP02_OPERATION_IDENTITY_BOUND: recover real disk records in new store context"
);
let recovered = recovered.expect("checked recovery result");
assert_eq!(
(
recovered.preparation_id(),
recovered.version_id(),
recovered.object_digest(),
recovered.object_length()
),
expected,
"NP02_OPERATION_IDENTITY_BOUND: recovery retains exact operation, version and content"
);
assert!(recovered.receipt_count() >= 3, "NP02_WDATA_BOUND: recovered write quorum");
assert_eq!(
recovered.disk_ids().collect::<HashSet<_>>().len(),
recovered.receipt_count(),
"NP02_WDATA_BOUND"
);
drop(recovered);
let replay = restarted
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await;
assert!(replay.is_ok(), "NP02_OPERATION_IDENTITY_BOUND: retry original preparation and content");
let replay = replay.expect("checked replay result");
assert_eq!(
(
replay.preparation_id(),
replay.version_id(),
replay.object_digest(),
replay.object_length()
),
expected,
"NP02_OPERATION_IDENTITY_BOUND: retry of the local preparation must not invent a new version"
);
drop(replay);
let mut changed_content = CONTENT.to_vec();
changed_content[0] ^= 1;
assert!(
restarted
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(changed_content))
.await
.is_err(),
"NP02_OPERATION_IDENTITY_BOUND: a local preparation cannot be rebound to different content"
);
assert!(
restarted
.recover_native_object(BUCKET, "another-key", preparation)
.await
.is_err(),
"NP02_OPERATION_IDENTITY_BOUND: recovery is bound to the original key"
);
}
#[tokio::test]
#[serial_test::serial]
async fn native_object_prepare_cancelled_writer_retains_pending_disk_owner() {
let (dirs, store) = fixture().await;
fail_preparation_disk(&dirs[0]);
let preparation = Uuid::new_v4();
let (entered, release) = crate::disk::local::native_prepare::pause_native_preparation(preparation, "PREPARED:file");
// LocalDisk canonicalizes its root before constructing metadata lock keys.
let owner_paths = dirs[1..]
.iter()
.map(|dir| {
std::fs::canonicalize(dir.path())
.expect("actual canonical disk root")
.join(BUCKET)
.join(KEY)
})
.collect::<Vec<_>>();
assert_eq!(owner_paths.len(), 3, "three healthy actual local disk owners");
let writing_store = store.clone();
let writer = tokio::spawn(async move {
writing_store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await
});
let reached_io = tokio::task::spawn_blocking(move || entered.recv_timeout(std::time::Duration::from_secs(30)))
.await
.expect("observe actual disk barrier");
// Release before asserting so a failed test never strands the real blocking I/O.
if reached_io.is_err() {
let _ = release.send(());
writer.abort();
let _ = writer.await;
crate::disk::local::native_prepare::clear_native_preparation_probe(preparation);
panic!("NP02_PENDING_OWNER_BOUND: preparation must reach actual disk I/O");
}
writer.abort();
let cancelled = matches!(writer.await, Err(error) if error.is_cancelled());
let mut held_owners = 0;
for path in &owner_paths {
if tokio::time::timeout(
std::time::Duration::from_millis(50),
crate::disk::os::acquire_metadata_mutation_lease(path, None),
)
.await
.is_err()
{
held_owners += 1;
}
}
release.send(()).expect("release actual disk I/O after cancelled waiter");
for path in &owner_paths {
let owner = tokio::time::timeout(
std::time::Duration::from_secs(5),
crate::disk::os::acquire_metadata_mutation_lease(path, None),
)
.await
.expect("completed actual I/O releases metadata owner");
drop(owner);
}
crate::disk::local::native_prepare::clear_native_preparation_probe(preparation);
assert!(
cancelled,
"NP02_PENDING_OWNER_BOUND: writer remains pending below W3 until the actual disk barrier"
);
assert!(
held_owners >= 1,
"NP02_PENDING_OWNER_BOUND: cancellation cannot release an in-flight real disk owner"
);
assert_not_published(&store).await;
drop(store);
let restarted = reopen(&dirs).await;
let recovered = restarted.recover_native_object(BUCKET, KEY, preparation).await;
assert!(
recovered.is_ok(),
"NP02_PENDING_OWNER_BOUND: retain actual durable facts after cancelled waiter"
);
let recovered = recovered.expect("checked recovery result");
let version = recovered.version_id();
assert_eq!(recovered.preparation_id(), preparation, "NP02_OPERATION_IDENTITY_BOUND");
assert_eq!(
recovered.object_digest(),
<[u8; 32]>::from(Sha256::digest(CONTENT)),
"NP02_OPERATION_IDENTITY_BOUND"
);
assert_eq!(recovered.object_length(), u64::try_from(CONTENT.len()).expect("small content"));
assert_eq!(recovered.receipt_count(), 3, "NP02_WDATA_BOUND");
drop(recovered);
let replay = restarted
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await;
assert!(
replay.is_ok(),
"NP02_OPERATION_IDENTITY_BOUND: retry cancelled operation from persisted facts"
);
assert_eq!(replay.expect("checked replay").version_id(), version, "NP02_OPERATION_IDENTITY_BOUND");
assert_not_published(&restarted).await;
}
#[tokio::test]
#[serial_test::serial]
async fn native_object_prepare_nonzero_set_reopens_the_routed_identity() {
let dirs = (0..8)
.map(|_| tempfile::tempdir().expect("real local disk directory"))
.collect::<Vec<_>>();
let store = reopen(&dirs).await;
store
.make_bucket(BUCKET, &MakeBucketOptions::default())
.await
.expect("real two-set bucket");
let selected_key = (0..1024)
.map(|index| format!("routed/{index}.bin"))
.find(|key| store.pools[0].get_disks_by_key(key).set_index == 1)
.expect("find a key routed by the actual algorithm to nonzero set");
let other_key = (0..1024)
.map(|index| format!("separate/{index}.bin"))
.find(|key| store.pools[0].get_disks_by_key(key).set_index == 0)
.expect("find another key routed to the other real set");
let target_disks = store.pools[0].format.erasure.sets[1].iter().copied().collect::<HashSet<_>>();
let preparation = Uuid::new_v4();
let prepared = store
.prepare_native_object(BUCKET, &selected_key, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
.await;
assert!(
prepared.is_ok(),
"NP02_OPERATION_IDENTITY_BOUND: nonzero routed set prepares real data: {:?}",
prepared.as_ref().err()
);
let prepared = prepared.expect("checked nonzero set preparation");
let expected = (
prepared.preparation_id(),
prepared.version_id(),
prepared.object_digest(),
prepared.object_length(),
);
assert!(
prepared.receipt_count() >= 3,
"NP02_WDATA_BOUND: quorum is scoped to the selected physical set"
);
let actual_disks = prepared.disk_ids().collect::<HashSet<_>>();
assert_eq!(actual_disks.len(), prepared.receipt_count(), "NP02_WDATA_BOUND");
assert!(
actual_disks.is_subset(&target_disks),
"NP02_OPERATION_IDENTITY_BOUND: receipts must come from routed set one"
);
drop(prepared);
assert!(
dirs[..4].iter().all(|dir| !dir
.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string())
.exists()),
"NP02_OPERATION_IDENTITY_BOUND: preparation must not write unrelated set zero"
);
let mut other_content = CONTENT.to_vec();
other_content[0] ^= 1;
let other = store
.prepare_native_object(BUCKET, &other_key, preparation, &mut PutObjReader::from_vec(other_content.clone()))
.await;
assert!(
other.is_ok(),
"NP02_OPERATION_IDENTITY_BOUND: preparation UUID is local set/key identity, not global operation authority"
);
drop(other);
for key in [&selected_key, &other_key] {
let head = store.get_object_info(BUCKET, key, &ObjectOptions::default()).await;
assert!(
matches!(head, Err(ref error) if crate::error::is_err_object_not_found(error)),
"NP02_PREPARATION_NOT_AUTHORITY"
);
}
drop(store);
let restarted = reopen(&dirs).await;
assert_eq!(
restarted.pools[0].get_disks_by_key(&selected_key).set_index,
1,
"NP02_OPERATION_IDENTITY_BOUND: durable routing after reopen"
);
let recovered = restarted.recover_native_object(BUCKET, &selected_key, preparation).await;
assert!(recovered.is_ok(), "NP02_OPERATION_IDENTITY_BOUND: reopen routed set preparation");
let recovered = recovered.expect("checked routed recovery");
assert_eq!(
(
recovered.preparation_id(),
recovered.version_id(),
recovered.object_digest(),
recovered.object_length()
),
expected,
"NP02_OPERATION_IDENTITY_BOUND: other set facts cannot replace selected key content"
);
assert!(
recovered.disk_ids().collect::<HashSet<_>>().is_subset(&target_disks),
"NP02_OPERATION_IDENTITY_BOUND"
);
drop(recovered);
let other = restarted.recover_native_object(BUCKET, &other_key, preparation).await;
assert!(
other.is_ok(),
"NP02_OPERATION_IDENTITY_BOUND: recover independent preparation in other set"
);
assert_eq!(
other.expect("checked other set recovery").object_digest(),
<[u8; 32]>::from(Sha256::digest(&other_content)),
"NP02_OPERATION_IDENTITY_BOUND: select actual key/set facts"
);
}
#[tokio::test]
#[serial_test::serial]
async fn native_stream_prepare_large_restart() {
let (dirs, store) = fixture().await;
let body = (0..(5 * 1024 * 1024 + 37))
.map(|index| u8::try_from(index % 251).expect("bounded byte"))
.collect::<Vec<_>>();
let expected: [u8; 32] = Sha256::digest(&body).into();
let preparation = Uuid::new_v4();
let prepared = store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(body.clone()))
.await
.expect("actual multi-block streaming preparation");
assert_eq!(prepared.object_digest(), expected);
assert_eq!(prepared.object_length(), u64::try_from(body.len()).expect("bounded content length"));
assert!(prepared.receipt_count() >= 3);
assert_not_published(&store).await;
let mut records = Vec::new();
for disk in &dirs {
let path = disk
.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string());
let metadata = std::fs::read(path.join("xl.meta")).expect("exact streaming metadata");
let record: serde_json::Value =
serde_json::from_slice(&std::fs::read(path.join("intent.json")).expect("durable locator"))
.expect("strict streaming intent JSON");
assert_eq!(record["format"], 3);
let disk_format: crate::layout::format::FormatV3 =
serde_json::from_slice(&std::fs::read(disk.path().join(".rustfs.sys/format.json")).expect("actual disk format"))
.expect("typed disk format");
let disk_id: Uuid = serde_json::from_value(record["disk"].clone()).expect("record disk identity");
let expected_set = disk_format
.erasure
.sets
.iter()
.find(|set| set.contains(&disk_id))
.expect("disk belongs to one ordered set");
let saved_set: Vec<Uuid> = serde_json::from_value(record["set_disks"].clone()).expect("persistent disk set");
assert_eq!(&saved_set, expected_set, "persist the full ordered physical set");
assert_eq!(record["data_dir"], preparation.to_string());
assert!(
std::fs::metadata(path.join(preparation.to_string()).join("part.1"))
.expect("real non-inline shard")
.len()
> 0
);
records.push((path, metadata));
}
drop(prepared);
drop(store);
let restarted = reopen(&dirs).await;
let recovered = restarted
.recover_native_object(BUCKET, KEY, preparation)
.await
.expect("actual streaming reopen recovery");
assert_eq!(recovered.object_digest(), expected);
assert_eq!(recovered.object_length(), u64::try_from(body.len()).expect("length"));
assert_eq!(recovered.version_id(), preparation);
for (path, bytes) in records {
assert_eq!(std::fs::read(path.join("xl.meta")).expect("preserved metadata"), bytes);
}
assert_not_published(&restarted).await;
}
#[tokio::test]
#[serial_test::serial]
async fn native_stream_prepare_locator_conflict() {
let (dirs, store) = fixture().await;
let preparation = Uuid::new_v4();
let body = vec![17; 2 * 1024 * 1024 + 1];
drop(
store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(body))
.await
.expect("real durable streaming preparation"),
);
for disk in &dirs {
let path = disk
.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string());
let mut record: serde_json::Value =
serde_json::from_slice(&std::fs::read(path.join("intent.json")).expect("record")).expect("JSON");
record["set_disks"]
.as_array_mut()
.expect("saved physical disk order")
.swap(0, 1);
// Preserve the actual record serialization order; receipt marker signs exact canonical record bytes.
let bytes = crate::disk::local::native_prepare::canonical_test_record(&record);
std::fs::write(path.join("intent.json"), &bytes).expect("install conflicting durable locator");
std::fs::write(path.join("PREPARED"), Sha256::digest(&bytes)).expect("matching record marker");
}
assert!(
store.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
"NATIVE_STREAM_REQUIRES_SAVED_PHYSICAL_SET"
);
assert_not_published(&store).await;
}
#[test]
#[serial_test::serial]
fn native_stream_prepare_locator_conflict_across_pools_and_unknown_disk() {
std::thread::Builder::new()
.stack_size(32 * 1024 * 1024)
.spawn(|| {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.worker_threads(2)
.thread_stack_size(32 * 1024 * 1024)
.build()
.expect("large-stack test runtime");
runtime.block_on(async {
use crate::disk::DiskAPI as _;
let (dirs, store) = multi_pool_fixture().await;
let preparation = Uuid::new_v4();
prepare_stream_in_pool(&store, 0, preparation, vec![41; 2 * 1024 * 1024 + 3]).await;
prepare_stream_in_pool(&store, 1, preparation, vec![42; 2 * 1024 * 1024 + 3]).await;
let pool_zero: serde_json::Value = serde_json::from_slice(
&std::fs::read(
dirs[0]
.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string())
.join("intent.json"),
)
.expect("pool zero durable locator"),
)
.expect("pool zero locator JSON");
let pool_one: serde_json::Value = serde_json::from_slice(
&std::fs::read(
dirs[4]
.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string())
.join("intent.json"),
)
.expect("pool one durable locator"),
)
.expect("pool one locator JSON");
assert_ne!(
pool_zero["object_digest"], pool_one["object_digest"],
"NATIVE_CROSS_POOL_CONFLICT_FIXTURE_MUST_HAVE_DIFFERENT_CONTENT"
);
assert!(
store.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
"NATIVE_STREAM_MUST_REJECT_PREPARATION_IN_MULTIPLE_POOLS"
);
assert_not_published(&store).await;
let unknown_preparation = Uuid::new_v4();
prepare_stream_in_pool(&store, 0, unknown_preparation, vec![51; 2 * 1024 * 1024 + 5]).await;
let unknown_disk = store.pools[0].get_disks_by_key(KEY).disks.read().await[0]
.as_ref()
.expect("pool zero physical disk")
.get_disk_id()
.await
.expect("read pool zero physical identity")
.expect("known pool zero physical identity");
let disk_root = dirs
.iter()
.find(|dir| {
serde_json::from_slice::<crate::layout::format::FormatV3>(
&std::fs::read(dir.path().join(".rustfs.sys/format.json")).expect("read physical format"),
)
.is_ok_and(|format| format.erasure.this == unknown_disk)
})
.expect("resolve pool zero physical root");
let prep_dir = disk_root
.path()
.join(".rustfs.sys/native-preparations")
.join(unknown_preparation.to_string());
let record_path = prep_dir.join("intent.json");
let mut record: serde_json::Value =
serde_json::from_slice(&std::fs::read(&record_path).expect("durable locator")).expect("locator JSON");
let unknown_identity = Uuid::new_v4();
assert_ne!(unknown_disk, unknown_identity);
record["disk"] = serde_json::Value::String(unknown_identity.to_string());
let bytes = crate::disk::local::native_prepare::canonical_test_record(&record);
std::fs::write(&record_path, &bytes).expect("install unknown disk identity");
std::fs::write(prep_dir.join("PREPARED"), Sha256::digest(&bytes))
.expect("retain valid marker for unknown identity");
assert!(
store.recover_native_object(BUCKET, KEY, unknown_preparation).await.is_err(),
"NATIVE_STREAM_MUST_FAIL_CLOSED_ON_UNKNOWN_DISK_IDENTITY"
);
assert!(
store
.prepare_native_object(
BUCKET,
KEY,
unknown_preparation,
&mut PutObjReader::from_vec(vec![51; 2 * 1024 * 1024 + 5]),
)
.await
.is_err(),
"NATIVE_STREAM_MUST_NOT_REALLOCATE_WHEN_DISK_IDENTITY_IS_UNKNOWN"
);
assert!(
dirs[4..].iter().all(|dir| !dir
.path()
.join(".rustfs.sys/native-preparations")
.join(unknown_preparation.to_string())
.exists()),
"NATIVE_STREAM_UNKNOWN_DISK_MUST_NOT_RECEIVE_REASSIGNED_PREPARATION"
);
assert_not_published(&store).await;
});
})
.expect("spawn large-stack native preparation test")
.join()
.expect("native preparation test thread");
}
#[tokio::test]
#[serial_test::serial]
async fn native_stream_prepare_crash_barriers() {
use crate::disk::local::native_prepare::{clear_native_preparation_probe, fail_native_preparation};
for (stage, after, recoverable) in [
("part.1:file", false, false),
("intent.json:file", true, false),
("PREPARED:directory", true, true),
] {
let (dirs, store) = fixture().await;
let preparation = Uuid::new_v4();
fail_native_preparation(preparation, stage, after);
let result = store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(vec![23; 2 * 1024 * 1024 + 3]))
.await;
assert!(result.is_err(), "injected disk barrier must not return a receipt: {stage}");
clear_native_preparation_probe(preparation);
assert_not_published(&store).await;
drop(store);
let restarted = reopen(&dirs).await;
let recovered = restarted.recover_native_object(BUCKET, KEY, preparation).await;
assert_eq!(recovered.is_ok(), recoverable, "durable boundary: {stage}");
assert_not_published(&restarted).await;
}
}
#[tokio::test]
#[serial_test::serial]
async fn native_stream_prepare_multi_part_restart() {
use crate::disk::Disk;
use rustfs_filemeta::FileInfoOpts;
let (dirs, store) = fixture().await;
let body = vec![29; 3 * 1024 * 1024 + 13];
let encoded_preparation = Uuid::new_v4();
drop(
store
.prepare_native_object(BUCKET, KEY, encoded_preparation, &mut PutObjReader::from_vec(body.clone()))
.await
.expect("existing ordinary streaming encoder fixture"),
);
let preparation = Uuid::new_v4();
let incarnation = store.bucket_incarnation_id(BUCKET).await.expect("actual incarnation");
let fence = store.object_lock_config_snapshot(BUCKET).await.expect("actual bucket fence");
let set = store.pools[0].get_disks_by_key(KEY);
let disks = set.disks.read().await.clone();
let guard = store
.handle_new_ns_lock(BUCKET, KEY)
.await
.expect("actual commit namespace")
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await
.expect("owned commit lock");
let mut candidates = Vec::new();
for (dir, disk) in dirs.iter().zip(disks) {
let original = dir
.path()
.join(".rustfs.sys/native-preparations")
.join(encoded_preparation.to_string());
let metadata = std::fs::read(original.join("xl.meta")).expect("encoder metadata");
let mut info = rustfs_filemeta::get_file_info(
&metadata,
BUCKET,
KEY,
&encoded_preparation.to_string(),
FileInfoOpts {
data: false,
include_free_versions: false,
include_part_checksums: true,
},
)
.expect("exact FileInfo");
let first = info.erasure.block_size;
let codec = crate::erasure::coding::Erasure::try_new_with_options(
info.erasure.data_blocks,
info.erasure.parity_blocks,
first,
info.uses_legacy_checksum,
)
.expect("existing erasure geometry");
let split = crate::erasure::coding::bitrot_shard_file_size(
usize::try_from(codec.shard_file_size(i64::try_from(first).expect("part size"))).expect("shard size"),
codec.shard_size(),
info.erasure.get_checksum_info(1).algorithm,
);
let shard = std::fs::read(original.join(encoded_preparation.to_string()).join("part.1")).expect("actual encoded shard");
let disk = disk.expect("actual disk");
let Disk::Local(local) = disk.as_ref() else {
panic!("local fixture");
};
local
.get_disk()
.begin_native_stream(preparation, store.id, BUCKET, incarnation, fence.clone())
.await
.expect("actual private owner");
let destination = dir
.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string())
.join(preparation.to_string());
std::fs::create_dir(&destination).expect("private part fixture directory");
std::fs::write(destination.join("part.1"), &shard[..split]).expect("first actual encoded blocks");
std::fs::write(destination.join("part.2"), &shard[split..]).expect("remaining actual encoded blocks");
info.version_id = Some(preparation);
info.data_dir = Some(preparation);
info.parts.clear();
info.add_object_part(1, String::new(), first, None, i64::try_from(first).expect("size"), None, None);
info.add_object_part(
2,
String::new(),
body.len() - first,
None,
i64::try_from(body.len() - first).expect("size"),
None,
None,
);
candidates.push((disk.clone(), info));
}
let owner = set
.attest_native_stream(
store.id,
BUCKET,
KEY,
preparation,
incarnation,
candidates,
Sha256::digest(&body).into(),
u64::try_from(body.len()).expect("length"),
fence,
)
.await
.expect("actual adoption verifies/fsyncs both parts without rereading upload or taking namespace");
assert!(owner.receipt_count() >= 3);
drop(owner);
drop(guard);
drop(set);
drop(store);
let restarted = reopen(&dirs).await;
let recovered = restarted
.recover_native_object(BUCKET, KEY, preparation)
.await
.expect("actual multi-part recovery decode");
assert_eq!(recovered.object_digest(), <[u8; 32]>::from(Sha256::digest(&body)));
assert_eq!(recovered.object_length(), u64::try_from(body.len()).expect("length"));
assert_not_published(&restarted).await;
}
#[tokio::test]
#[serial_test::serial]
async fn native_stream_prepare_rejects_overwrite() {
use crate::disk::{DiskAPI, RUSTFS_META_BUCKET};
let (dirs, store) = fixture().await;
let preparation = Uuid::new_v4();
drop(
store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(vec![31; 2 * 1024 * 1024 + 11]))
.await
.expect("attested private stream"),
);
let disk = store.pools[0].get_disks_by_key(KEY).disks.read().await[0]
.clone()
.expect("actual disk");
let path = format!("native-preparations/{preparation}/{preparation}/part.1");
let full = dirs[0].path().join(RUSTFS_META_BUCKET).join(&path);
let before = std::fs::read(&full).expect("attested shard bytes");
assert!(
disk.create_file("", RUSTFS_META_BUCKET, &path, 1).await.is_err(),
"NATIVE_STREAM_ATTESTED_PART_MUST_NOT_BE_TRUNCATED"
);
assert_eq!(std::fs::read(&full).expect("retained shard"), before);
assert!(store.recover_native_object(BUCKET, KEY, preparation).await.is_ok());
}
#[tokio::test]
#[serial_test::serial]
async fn native_stream_prepare_record_locator() {
use crate::disk::Disk;
let (dirs, store) = fixture().await;
let preparation = Uuid::new_v4();
drop(
store
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(vec![37; 2 * 1024 * 1024 + 9]))
.await
.expect("actual prepared locator"),
);
let disk = store.pools[0].get_disks_by_key(KEY).disks.read().await[0]
.clone()
.expect("actual disk");
let Disk::Local(local) = disk.as_ref() else {
panic!("local fixture");
};
let listed = local
.get_disk()
.list_native_preparations(store.id, None, 1)
.await
.expect("descriptor-relative list");
assert_eq!(listed.records.len(), 1);
assert!(listed.unfinished.is_empty());
assert_eq!(listed.next, None);
let record = &listed.records[0];
assert_eq!(record.preparation, preparation);
assert_eq!(record.bucket, BUCKET);
assert_eq!(record.key, KEY);
assert_eq!(record.version, preparation);
assert!(record.prepared);
let path = dirs[0]
.path()
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string());
assert_eq!(
record.metadata.as_ref(),
std::fs::read(path.join("xl.meta")).expect("retained exact metadata")
);
let direct = local
.get_disk()
.locate_native_preparation(store.id, preparation)
.await
.expect("direct locator")
.expect("present record");
assert_eq!(direct.metadata, record.metadata);
assert!(
local
.get_disk()
.locate_native_preparation(store.id, Uuid::new_v4())
.await
.expect("healthy absence")
.is_none()
);
assert!(local.get_disk().list_native_preparations(store.id, None, 65).await.is_err());
// Sender-owned namespace guard must not be acquired again by the recovery receiver.
let guard = store
.handle_new_ns_lock(BUCKET, KEY)
.await
.expect("namespace")
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await
.expect("actual sender lock");
let (owner, metadata) = tokio::time::timeout(
std::time::Duration::from_secs(10),
store.recover_native_object_unlocked(BUCKET, KEY, preparation),
)
.await
.expect("no recursive namespace lock")
.expect("actual disk recovery under sender lock");
assert!(metadata.iter().any(|(id, bytes)| {
owner
.receipts()
.iter()
.any(|receipt| receipt.disk_id() == *id && receipt.metadata_digest() == &<[u8; 32]>::from(Sha256::digest(bytes)))
}));
drop(guard);
// Moving a published part must not erase its durable location facts; locate grants no readiness.
std::fs::rename(
path.join(preparation.to_string()).join("part.1"),
dirs[0].path().join("published-part-test"),
)
.expect("publication progress fixture");
let located = local
.get_disk()
.locate_native_preparation(store.id, preparation)
.await
.expect("retained locator after part move")
.expect("retained record");
assert_eq!(located.metadata, record.metadata);
assert!(
store.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
"prepared marker is not publication authority"
);
}
+219
View File
@@ -0,0 +1,219 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Durable restrictions owned by an authenticated user. A record is not an IAM
//! grant or credential; issuing credentials still requires current IAM checks.
use crate::{error::is_err_config_not_found, store::Store};
use serde::{Deserialize, Serialize};
const PREFIX: &str = "config/iam/plugin-delegations/v1";
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct Restrictions {
pub deployment: String,
pub bucket: String,
pub input_prefix: String,
pub output_prefix: String,
pub expires_at_unix_seconds: u64,
}
impl Restrictions {
pub fn validate(&self, now: u64) -> Result<(), DelegationError> {
let names = [&self.deployment, &self.bucket, &self.input_prefix, &self.output_prefix];
if names
.iter()
.any(|s| s.is_empty() || s.len() > 1024 || s.contains("${") || s.chars().any(char::is_control))
|| self.bucket.contains('/')
|| self.bucket.contains('*')
|| self.bucket.contains('?')
|| self.input_prefix.contains('*')
|| self.input_prefix.contains('?')
|| self.output_prefix.contains('*')
|| self.output_prefix.contains('?')
|| self.expires_at_unix_seconds <= now
|| self.expires_at_unix_seconds.saturating_sub(now) > 86400
{
return Err(DelegationError::Invalid);
}
Ok(())
}
}
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct Delegation {
pub version: u32,
pub id: String,
pub revision: String,
pub owner_access_key: String,
pub created_at_unix_seconds: u64,
pub restrictions: Restrictions,
}
#[derive(Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
struct Tombstone {
version: u32,
id: String,
revision: String,
owner_access_key: String,
}
#[derive(Debug, thiserror::Error)]
pub enum DelegationError {
#[error("invalid delegation restriction")]
Invalid,
#[error("delegation not found")]
NotFound,
#[error("delegation owner rejected")]
OwnerRejected,
#[error("delegation unavailable")]
Unavailable,
#[error("delegation revoked")]
Revoked,
#[error("delegation expired")]
Expired,
}
fn record_path(id: &str) -> Result<String, DelegationError> {
if id.len() != 32 || !id.bytes().all(|c| c.is_ascii_hexdigit()) {
return Err(DelegationError::Invalid);
}
Ok(format!("{PREFIX}/{id}/record.json"))
}
async fn load<T: Store>(store: &T, id: &str) -> Result<Delegation, DelegationError> {
let record: Delegation = store.load_iam_config_strict(record_path(id)?).await.map_err(|error| {
if is_err_config_not_found(&error) {
DelegationError::NotFound
} else {
DelegationError::Unavailable
}
})?;
if record.version != 1 || record.id != id || record.revision != id || record.owner_access_key.is_empty() {
return Err(DelegationError::Unavailable);
}
record
.restrictions
.validate(record.created_at_unix_seconds)
.map_err(|_| DelegationError::Unavailable)?;
Ok(record)
}
pub(crate) async fn create<T: Store>(
store: &T,
owner: &str,
restrictions: Restrictions,
now: u64,
) -> Result<Delegation, DelegationError> {
restrictions.validate(now)?;
if owner.is_empty() {
return Err(DelegationError::OwnerRejected);
}
// IDs are generated exclusively here; no update operation accepts an ID.
let id = format!("{:032x}", rand::random::<u128>());
let path = record_path(&id)?;
match store.load_iam_config_strict::<Delegation>(&path).await {
Err(error) if is_err_config_not_found(&error) => {}
_ => return Err(DelegationError::Unavailable),
}
let record = Delegation {
version: 1,
revision: id.clone(),
id,
owner_access_key: owner.to_owned(),
created_at_unix_seconds: now,
restrictions,
};
store
.save_iam_config(&record, path)
.await
.map_err(|_| DelegationError::Unavailable)?;
if load(store, &record.id).await? != record {
return Err(DelegationError::Unavailable);
}
Ok(record)
}
pub(crate) async fn revoke<T: Store>(store: &T, owner: &str, id: &str) -> Result<(), DelegationError> {
let record = load(store, id).await?;
if record.owner_access_key != owner {
return Err(DelegationError::OwnerRejected);
}
let tombstone = Tombstone {
version: 1,
id: record.id,
revision: record.revision,
owner_access_key: record.owner_access_key,
};
let path = format!("{PREFIX}/{id}/revoked.json");
// Every retry writes exactly the same revocation; active records never change.
store
.save_iam_config(&tombstone, &path)
.await
.map_err(|_| DelegationError::Unavailable)?;
let saved: Tombstone = store
.load_iam_config_strict(path)
.await
.map_err(|_| DelegationError::Unavailable)?;
if saved != tombstone {
return Err(DelegationError::Unavailable);
}
Ok(())
}
pub(crate) async fn read_active<T: Store>(store: &T, owner: &str, id: &str, now: u64) -> Result<Delegation, DelegationError> {
let record = load(store, id).await?;
if record.owner_access_key != owner {
return Err(DelegationError::OwnerRejected);
}
check_active(store, record, now).await
}
pub(crate) async fn read_active_for_broker<T: Store>(store: &T, id: &str, now: u64) -> Result<Delegation, DelegationError> {
let record = load(store, id).await?;
check_active(store, record, now).await
}
async fn check_active<T: Store>(store: &T, record: Delegation, now: u64) -> Result<Delegation, DelegationError> {
let id = &record.id;
match store
.load_iam_config_strict::<Tombstone>(format!("{PREFIX}/{id}/revoked.json"))
.await
{
Ok(_) => return Err(DelegationError::Revoked),
Err(error) if is_err_config_not_found(&error) => {}
Err(_) => return Err(DelegationError::Unavailable),
}
if now >= record.restrictions.expires_at_unix_seconds {
return Err(DelegationError::Expired);
}
Ok(record)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn restrictions_are_literal_bounded_and_expiring() {
let mut restriction = Restrictions {
deployment: "runner-a".into(),
bucket: "photos".into(),
input_prefix: "inputs/".into(),
output_prefix: "outputs/".into(),
expires_at_unix_seconds: 200,
};
assert!(restriction.validate(100).is_ok());
restriction.output_prefix = "outputs/*".into();
assert!(matches!(restriction.validate(100), Err(DelegationError::Invalid)));
restriction.output_prefix = "outputs/".into();
assert!(matches!(restriction.validate(200), Err(DelegationError::Invalid)));
restriction.expires_at_unix_seconds = 200;
restriction.input_prefix = "inputs/${aws:username}/".into();
assert!(
matches!(restriction.validate(100), Err(DelegationError::Invalid)),
"C07_LITERAL_SCOPE_BOUND: policy variables are not literal restrictions"
);
assert!(record_path("../../users").is_err());
}
}
+1
View File
@@ -28,6 +28,7 @@ pub(crate) const EVENT_OIDC_DIAGNOSTICS: &str = "oidc_diagnostics";
const EVENT_OIDC_STATE: &str = "oidc_state";
pub mod cache;
pub mod delegation;
pub mod error;
pub mod federation;
pub mod keyring;
+5
View File
@@ -26,6 +26,11 @@ pub trait Store: Clone + Send + Sync + 'static {
fn has_watcher(&self) -> bool;
async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()>;
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item>;
/// Only confirmed storage absence may return ConfigNotFound. Corrupt or
/// undecryptable objects must remain errors, especially for revocations.
async fn load_iam_config_strict<Item: DeserializeOwned>(&self, _path: impl AsRef<str> + Send) -> Result<Item> {
Err(crate::error::Error::IamSysNotInitialized)
}
async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;
+46
View File
@@ -617,6 +617,11 @@ impl ObjectStore {
Ok(serde_json::from_slice(&outcome.plain)?)
}
fn decode_iam_config_strict<Item: DeserializeOwned>(data: &[u8]) -> Result<Item> {
let outcome = Self::decrypt_data_with_source(data)?;
Ok(serde_json::from_slice(&outcome.plain)?)
}
/// Parameterized core of [`Store::load_user_identity`].
async fn load_user_identity_with(&self, name: &str, user_type: UserType, mode: LoadMode) -> Result<UserIdentity> {
let mut u: UserIdentity = self
@@ -831,6 +836,13 @@ impl Store for ObjectStore {
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
self.load_iam_config_with(path, LoadMode::Locked).await
}
async fn load_iam_config_strict<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
let (data, _) =
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path.as_ref(), &LoadMode::Locked.read_opts())
.await?;
Self::decode_iam_config_strict(&data)
}
/// Saves IAM configuration with a retry mechanism on failure.
///
/// Attempts to save the IAM configuration up to 5 times if the storage layer is not ready,
@@ -1465,6 +1477,40 @@ mod tests {
assert!(result.is_err(), "corrupt stream_io data should fail decrypt");
}
#[test]
fn strict_revocation_load_keeps_real_crypto_failure_distinct_from_absence() {
let cred = test_cred();
let key = format!("{}:{}", cred.access_key, cred.secret_key);
let plain = br#"{"version":1,"id":"revoked"}"#;
let mut encrypted = rustfs_crypto::encrypt_stream_io(key.as_bytes(), plain).expect("real stored ciphertext");
// Exercise the exact decoder used after the actual object read.
assert!(ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted).is_ok());
encrypted[50] ^= 0xFF;
let error = ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted)
.expect_err("corrupt tombstone must not become missing");
assert!(
matches!(error, Error::CryptoError(_)),
"C07_REVOCATION_FAIL_CLOSED: preserve actual crypto error"
);
assert!(
!crate::error::is_err_config_not_found(&error),
"C07_REVOCATION_FAIL_CLOSED: corrupted revocation is not absence"
);
// The preserve-empty reader keeps an existing zero-byte object as
// Ok(empty); ecstore's read_config_preserve_empty_distinguishes_empty_object
// covers that storage boundary. Its payload must fail here, not become
// ConfigNotFound and reactivate a revoked delegation.
let empty =
ObjectStore::decode_iam_config_strict::<serde_json::Value>(&[]).expect_err("an existing empty tombstone is corrupt");
assert!(
!crate::error::is_err_config_not_found(&empty),
"C07_REVOCATION_FAIL_CLOSED: empty existing revocation is not absence"
);
let invalid_json =
ObjectStore::decode_iam_config_strict::<serde_json::Value>(br#"{"version": }"#).expect_err("invalid stored JSON");
assert!(!crate::error::is_err_config_not_found(&invalid_json));
}
#[test]
fn test_decrypt_data_short_data_fails() {
let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON
+249 -7
View File
@@ -75,6 +75,7 @@ pub const STATUS_ENABLED: &str = "enabled";
pub const STATUS_DISABLED: &str = "disabled";
pub const POLICYNAME: &str = "policy";
pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-policy";
@@ -168,6 +169,12 @@ enum PreparedServicePolicyMode {
#[derive(Clone)]
enum PreparedIamMode {
TerminalDeny,
PluginAttemptSts {
is_owner: bool,
combined_policy: Policy,
session_policy: PreparedSessionPolicy,
},
Opa,
Owner,
Deny,
@@ -202,7 +209,7 @@ impl PreparedIamAuth {
pub async fn needs_existing_object_tag_for_args(&self, args: &Args<'_>) -> bool {
match &self.mode {
PreparedIamMode::Opa => true,
PreparedIamMode::Owner | PreparedIamMode::Deny => false,
PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => false,
PreparedIamMode::Regular { combined_policy } => {
policy_needs_existing_object_tag_for_args(combined_policy, args).await
}
@@ -210,6 +217,11 @@ impl PreparedIamAuth {
combined_policy,
session_policy,
..
}
| PreparedIamMode::PluginAttemptSts {
combined_policy,
session_policy,
..
} => {
policy_needs_existing_object_tag_for_args(combined_policy, args).await
|| prepared_session_policy_needs_existing_object_tag_for_args(session_policy, args).await
@@ -239,14 +251,48 @@ impl PreparedIamAuth {
pub fn combined_policy_for_view(&self) -> Option<&Policy> {
match &self.mode {
PreparedIamMode::Regular { combined_policy } => Some(combined_policy),
PreparedIamMode::Sts { combined_policy, .. } => Some(combined_policy),
PreparedIamMode::Sts { combined_policy, .. } | PreparedIamMode::PluginAttemptSts { combined_policy, .. } => {
Some(combined_policy)
}
PreparedIamMode::ServiceAccount { combined_policy, .. } => Some(combined_policy),
PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny => None,
PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => None,
}
}
}
impl<T: Store> IamSys<T> {
/// For the authenticated, explicitly configured broker handler only.
pub async fn read_active_plugin_delegation_for_broker(
&self,
id: &str,
now: u64,
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
crate::delegation::read_active_for_broker(&self.store.api, id, now).await
}
pub async fn create_plugin_delegation(
&self,
owner: &str,
restrictions: crate::delegation::Restrictions,
now: u64,
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
crate::delegation::create(&self.store.api, owner, restrictions, now).await
}
pub async fn revoke_plugin_delegation(
&self,
owner: &str,
id: &str,
) -> std::result::Result<(), crate::delegation::DelegationError> {
crate::delegation::revoke(&self.store.api, owner, id).await
}
pub async fn read_active_plugin_delegation(
&self,
owner: &str,
id: &str,
now: u64,
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
crate::delegation::read_active(&self.store.api, owner, id, now).await
}
/// Create a new IamSys instance with the given IamCache store
///
/// # Arguments
@@ -281,6 +327,16 @@ impl<T: Store> IamSys<T> {
*guard = PolicyPluginState::Ready(client);
}
/// Restricted credentials require the built-in parent/session intersection.
/// External authorization bypasses that path, so only Disabled is supported.
pub async fn ensure_restricted_credential_vending() -> Result<()> {
if matches!(Self::policy_plugin_state().await, PolicyPluginState::Disabled) {
Ok(())
} else {
Err(Error::IAMActionNotAllowed)
}
}
pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
let policy_plugin_state = get_policy_plugin_state();
let guard = policy_plugin_state.read().await;
@@ -1383,17 +1439,50 @@ impl<T: Store> IamSys<T> {
/// 1) know whether policy evaluation may need `s3:ExistingObjectTag`, and
/// 2) evaluate with final conditions without re-merging identity policies.
pub async fn prepare_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
if args.is_owner {
let marked = args.claims.contains_key(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
if args.is_owner && !marked {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Owner,
};
}
if let Some(prepared) = Self::policy_plugin_state().await.prepared_iam_auth() {
// One observed state decides this request's path. Marked credentials
// never enter OPA, including after a restart or on another node.
let state = Self::policy_plugin_state().await;
if marked {
if args.is_owner
|| args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
|| !matches!(state, PolicyPluginState::Disabled)
{
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::TerminalDeny,
};
}
let prepared = self.prepare_builtin_auth(args).await;
let mode = match prepared.mode {
PreparedIamMode::Sts {
is_owner,
combined_policy,
session_policy,
} => PreparedIamMode::PluginAttemptSts {
is_owner,
combined_policy,
session_policy,
},
_ => PreparedIamMode::TerminalDeny,
};
return PreparedIamAuth {
needs_existing_object_tag: prepared.needs_existing_object_tag,
mode,
};
}
if let Some(prepared) = state.prepared_iam_auth() {
return prepared;
}
self.prepare_builtin_auth(args).await
}
async fn prepare_builtin_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
return PreparedIamAuth {
needs_existing_object_tag: false,
@@ -1425,6 +1514,20 @@ impl<T: Store> IamSys<T> {
/// bucket policies on an IAM denial. An error must not become an implicit deny.
pub async fn try_eval_prepared(&self, prepared: &PreparedIamAuth, args: &Args<'_>) -> Result<bool> {
Ok(match &prepared.mode {
PreparedIamMode::TerminalDeny => return Err(Error::IAMActionNotAllowed),
PreparedIamMode::PluginAttemptSts {
is_owner,
combined_policy,
session_policy,
} => {
// A false IAM result permits bucket-policy fallback. Restricted
// plugin credentials must instead terminate authorization.
let session_allowed = evaluate_prepared_session_policy(session_policy, args).await == Some(true);
if !session_allowed || !(*is_owner || combined_policy.is_allowed(args).await) {
return Err(Error::IAMActionNotAllowed);
}
true
}
PreparedIamMode::Opa => {
let Some(opa_enable) = Self::get_policy_plugin_client().await else {
tracing::warn!("eval_prepared: OPA mode requested but plugin is unavailable");
@@ -2127,6 +2230,145 @@ mod tests {
assert!(prepared.combined_policy_for_view().is_none());
}
#[tokio::test]
#[serial]
async fn marked_signed_sts_denial_is_terminal_before_public_bucket_fallback() {
let iam = IamSys::new(IamCache::new(StsTestMockStore::new(false)).await.unwrap());
let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
let signing_key = "c07-test-token-signing-key";
let mut metadata=HashMap::from([
("parent".into(),Value::String("sts-fallback-test-parent".into())),
("exp".into(),serde_json::json!((OffsetDateTime::now_utc()+time::Duration::hours(1)).unix_timestamp())),
(SESSION_POLICY_NAME.into(),Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/inside"]}]}"#))),
]);
metadata.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), Value::Bool(true));
let mut marked = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
marked.parent_user = "sts-fallback-test-parent".into();
let marked_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&marked.session_token, signing_key)
.expect("signed test credentials must decode")
.claims;
let marked_user = UserIdentity::from(marked.clone());
iam.store
.cache
.add_or_update_sts_account(&marked.access_key, &marked_user, OffsetDateTime::now_utc());
metadata.remove(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
let mut ordinary = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
ordinary.parent_user = marked.parent_user.clone();
let ordinary_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&ordinary.session_token, signing_key)
.expect("signed test credentials must decode")
.claims;
iam.store.cache.add_or_update_sts_account(
&ordinary.access_key,
&UserIdentity::from(ordinary.clone()),
OffsetDateTime::now_utc(),
);
let groups = None;
let conditions = HashMap::new();
let args = Args {
account: &marked.access_key,
groups: &groups,
action: Action::S3Action(S3Action::GetObjectAction),
bucket: "mybucket",
object: "outside",
conditions: &conditions,
is_owner: false,
claims: &marked_claims,
deny_only: false,
};
*get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
let denied = iam.prepare_auth(&args).await;
let disabled_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
let inside = Args {
object: "inside",
..args.clone()
};
let allowed = iam.prepare_auth(&inside).await;
let inside_allowed = matches!(iam.try_eval_prepared(&allowed, &inside).await, Ok(true));
let unmarked_args = Args {
account: &ordinary.access_key,
claims: &ordinary_claims,
..args.clone()
};
let unmarked = iam.prepare_auth(&unmarked_args).await;
let ordinary_false = matches!(iam.try_eval_prepared(&unmarked, &unmarked_args).await, Ok(false));
let public:rustfs_policy::policy::BucketPolicy=serde_json::from_value(serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/*"]}]})).unwrap();
let public_allows = public
.is_allowed(&rustfs_policy::policy::BucketPolicyArgs {
account: &marked.access_key,
groups: &groups,
action: args.action,
bucket: args.bucket,
object: args.object,
conditions: &conditions,
is_owner: false,
})
.await;
IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
url: "http://127.0.0.1:8181/allow".into(),
auth_token: String::new(),
}))
.await;
let opa_marked = iam.prepare_auth(&args).await;
let ready_terminal = matches!(iam.try_eval_prepared(&opa_marked, &args).await, Err(Error::IAMActionNotAllowed));
let opa_ordinary = iam.prepare_auth(&unmarked_args).await;
let ordinary_keeps_opa = matches!(opa_ordinary.mode, PreparedIamMode::Opa);
// The captured Disabled preparation stays on built-in intersection even
// if an external policy client becomes ready before evaluation.
let captured_still_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
*get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
let pending = iam.prepare_auth(&args).await;
let initializing_terminal = matches!(iam.try_eval_prepared(&pending, &args).await, Err(Error::IAMActionNotAllowed));
*get_policy_plugin_state().write().await = PolicyPluginState::Failed;
let failed = iam.prepare_auth(&args).await;
let failed_terminal = matches!(iam.try_eval_prepared(&failed, &args).await, Err(Error::IAMActionNotAllowed));
*get_policy_plugin_state().write().await = previous;
assert!(public_allows, "C07_PUBLIC_BUCKET_CONTROL: real public policy permits outside object");
assert!(
disabled_terminal && captured_still_terminal,
"C07_SCOPE_TERMINAL_BOUND: bucket policy must not rescue restricted scope denial"
);
assert!(
ready_terminal && initializing_terminal && failed_terminal,
"C07_EXTERNAL_POLICY_BOUND: marked persisted STS cannot enter OPA"
);
assert!(
inside_allowed && ordinary_false && ordinary_keeps_opa,
"C07_UNMARKED_COMPAT: preserve ordinary STS and valid marked scope"
);
}
#[tokio::test]
#[serial]
async fn restricted_vending_rejects_actual_external_policy_plugin_state() {
let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
url: "http://127.0.0.1:8181/v1/data/rustfs/authz/allow".into(),
auth_token: String::new(),
}))
.await;
let active_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
.await
.is_err();
*get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
let initializing_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
.await
.is_err();
*get_policy_plugin_state().write().await = PolicyPluginState::Failed;
let failed_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
.await
.is_err();
*get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
let disabled_allowed = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
.await
.is_ok();
*get_policy_plugin_state().write().await = previous;
assert!(
active_rejected && initializing_rejected && failed_rejected,
"C07_EXTERNAL_POLICY_BOUND: external authorization cannot bypass minted session scope"
);
assert!(disabled_allowed);
}
#[tokio::test]
#[serial]
async fn test_prepare_auth_requests_existing_object_tags_in_opa_mode() {
+31
View File
@@ -562,6 +562,14 @@ pub enum AdminAction {
GetTableLifecycleAction,
#[strum(serialize = "admin:SetTableLifecycle")]
SetTableLifecycleAction,
#[strum(serialize = "admin:CreatePluginDelegation")]
CreatePluginDelegationAction,
#[strum(serialize = "admin:IssuePluginAttempt")]
IssuePluginAttemptAction,
#[strum(serialize = "admin:GetPluginDelegation")]
GetPluginDelegationAction,
#[strum(serialize = "admin:RevokePluginDelegation")]
RevokePluginDelegationAction,
#[strum(serialize = "admin:GetTableCredentials")]
GetTableCredentialsAction,
#[strum(serialize = "admin:RunTableMaintenance")]
@@ -703,6 +711,10 @@ impl AdminAction {
| AdminAction::GetTableLifecycleAction
| AdminAction::SetTableLifecycleAction
| AdminAction::GetTableCredentialsAction
| AdminAction::IssuePluginAttemptAction
| AdminAction::GetPluginDelegationAction
| AdminAction::CreatePluginDelegationAction
| AdminAction::RevokePluginDelegationAction
| AdminAction::RunTableMaintenanceAction
| AdminAction::GetTableMetadataLocationAction
| AdminAction::SetTableMetadataLocationAction
@@ -903,6 +915,25 @@ mod tests {
assert_eq!(arr[0].as_str().unwrap(), "s3:*");
}
#[test]
fn plugin_delegation_actions_are_explicit_and_unscoped() {
for (name, expected) in [
("admin:GetPluginDelegation", AdminAction::GetPluginDelegationAction),
("admin:CreatePluginDelegation", AdminAction::CreatePluginDelegationAction),
("admin:RevokePluginDelegation", AdminAction::RevokePluginDelegationAction),
] {
assert_eq!(AdminAction::try_from(name).unwrap(), expected);
assert!(expected.is_valid());
assert!(!expected.is_table_resource_scoped());
}
assert_eq!(
AdminAction::try_from("admin:IssuePluginAttempt").unwrap(),
AdminAction::IssuePluginAttemptAction
);
assert!(AdminAction::IssuePluginAttemptAction.is_valid());
assert!(AdminAction::try_from("admin:IssuePluginCredentials").is_err());
}
#[test]
fn test_get_replication_metrics_admin_action_is_valid() {
assert!(AdminAction::GetReplicationMetricsAction.is_valid());
@@ -156,6 +156,217 @@ pub struct CompareAndUpdateFileResponse {
#[prost(message, optional, tag = "3")]
pub error: ::core::option::Option<Error>,
}
/// Native preparation carries only bounded object metadata over the control RPC.
/// Shard bytes use the authenticated data stream and remain private until finalized.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationReceipt {
#[prost(bytes = "bytes", tag = "1")]
pub disk_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "2")]
pub store_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "3")]
pub deployment_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "4")]
pub preparation_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "5")]
pub version_id: ::prost::bytes::Bytes,
#[prost(string, tag = "6")]
pub bucket: ::prost::alloc::string::String,
#[prost(string, tag = "7")]
pub key: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "8")]
pub bucket_incarnation_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "9")]
pub object_digest: ::prost::bytes::Bytes,
#[prost(uint64, tag = "10")]
pub object_length: u64,
#[prost(bytes = "bytes", tag = "11")]
pub set_digest: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "12")]
pub distribution_digest: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "13")]
pub metadata_digest: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "14")]
pub shard_digest: ::prost::bytes::Bytes,
#[prost(uint64, tag = "15")]
pub shard_length: u64,
#[prost(uint32, tag = "16")]
pub data_blocks: u32,
#[prost(uint32, tag = "17")]
pub parity_blocks: u32,
#[prost(uint32, tag = "18")]
pub shard_index: u32,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationBeginRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "2")]
pub disk_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "3")]
pub store_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "4")]
pub preparation_id: ::prost::bytes::Bytes,
#[prost(string, tag = "5")]
pub bucket: ::prost::alloc::string::String,
#[prost(string, tag = "6")]
pub key: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "7")]
pub bucket_incarnation_id: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationBeginResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(message, optional, tag = "2")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "3")]
pub response_proof: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationFinalizeRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "2")]
pub disk_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "3")]
pub store_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "4")]
pub preparation_id: ::prost::bytes::Bytes,
#[prost(string, tag = "5")]
pub bucket: ::prost::alloc::string::String,
#[prost(string, tag = "6")]
pub key: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "7")]
pub bucket_incarnation_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "8")]
pub object_digest: ::prost::bytes::Bytes,
#[prost(uint64, tag = "9")]
pub object_length: u64,
#[prost(bytes = "bytes", tag = "10")]
pub version_id: ::prost::bytes::Bytes,
/// Exact serialized FileMeta; maximum 1 MiB. Peers reject larger values.
#[prost(bytes = "bytes", tag = "11")]
pub metadata: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationFinalizeResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(message, optional, tag = "2")]
pub receipt: ::core::option::Option<NativePreparationReceipt>,
#[prost(message, optional, tag = "3")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "4")]
pub response_proof: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationProbeRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "2")]
pub disk_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "3")]
pub store_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "4")]
pub preparation_id: ::prost::bytes::Bytes,
#[prost(string, tag = "5")]
pub bucket: ::prost::alloc::string::String,
#[prost(string, tag = "6")]
pub key: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "7")]
pub bucket_incarnation_id: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationProbeResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(bool, tag = "2")]
pub exists: bool,
#[prost(bool, tag = "3")]
pub prepared: bool,
/// Exact serialized FileMeta when it exists and can be read; maximum 1 MiB.
#[prost(bytes = "bytes", tag = "5")]
pub metadata: ::prost::bytes::Bytes,
#[prost(message, optional, tag = "6")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "7")]
pub response_proof: ::prost::bytes::Bytes,
/// Observation only; this is not a durability receipt.
#[prost(bytes = "bytes", tag = "8")]
pub version_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "9")]
pub object_digest: ::prost::bytes::Bytes,
#[prost(uint64, tag = "10")]
pub object_length: u64,
#[prost(bytes = "bytes", tag = "11")]
pub set_digest: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "12")]
pub distribution_digest: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationRecoverRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "2")]
pub disk_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "3")]
pub store_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "4")]
pub preparation_id: ::prost::bytes::Bytes,
#[prost(string, tag = "5")]
pub bucket: ::prost::alloc::string::String,
#[prost(string, tag = "6")]
pub key: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "7")]
pub bucket_incarnation_id: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationRecoverResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(message, optional, tag = "2")]
pub receipt: ::core::option::Option<NativePreparationReceipt>,
/// Exact persisted FileMeta; maximum 1 MiB. Peers reject larger values.
#[prost(bytes = "bytes", tag = "3")]
pub metadata: ::prost::bytes::Bytes,
#[prost(message, optional, tag = "4")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "5")]
pub response_proof: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationLocateRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "2")]
pub disk_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "3")]
pub store_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "4")]
pub preparation_id: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NativePreparationLocateResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(bool, tag = "2")]
pub found: bool,
#[prost(string, tag = "3")]
pub bucket: ::prost::alloc::string::String,
#[prost(string, tag = "4")]
pub key: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "5")]
pub bucket_incarnation_id: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "6")]
pub metadata: ::prost::bytes::Bytes,
#[prost(bool, tag = "7")]
pub prepared: bool,
#[prost(message, optional, tag = "8")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "9")]
pub response_proof: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct DeleteRequest {
/// indicate which one in the disks
@@ -1692,6 +1903,19 @@ pub struct GetLiveEventsResponse {
#[prost(string, optional, tag = "5")]
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
}
/// Capture authority uses a bounded, versioned canonical body, not protobuf re-encoding.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CaptureRaftRequest {
#[prost(bytes = "bytes", tag = "1")]
pub body: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CaptureRaftResponse {
#[prost(bytes = "bytes", tag = "1")]
pub body: ::prost::bytes::Bytes,
#[prost(bytes = "bytes", tag = "2")]
pub response_proof: ::prost::bytes::Bytes,
}
/// Typed control-plane error discriminants carried alongside the legacy
/// error_info string on control-plane responses. Rolling-upgrade compat:
/// old peers ignore the field and keep reading error_info.
@@ -2052,6 +2276,81 @@ pub mod node_service_client {
.insert(GrpcMethod::new("node_service.NodeService", "CompareAndUpdateFile"));
self.inner.unary(req, path, codec).await
}
pub async fn native_preparation_begin(
&mut self,
request: impl tonic::IntoRequest<super::NativePreparationBeginRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationBeginResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/NativePreparationBegin");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "NativePreparationBegin"));
self.inner.unary(req, path, codec).await
}
pub async fn native_preparation_finalize(
&mut self,
request: impl tonic::IntoRequest<super::NativePreparationFinalizeRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationFinalizeResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/NativePreparationFinalize");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "NativePreparationFinalize"));
self.inner.unary(req, path, codec).await
}
pub async fn native_preparation_probe(
&mut self,
request: impl tonic::IntoRequest<super::NativePreparationProbeRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationProbeResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/NativePreparationProbe");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "NativePreparationProbe"));
self.inner.unary(req, path, codec).await
}
pub async fn native_preparation_recover(
&mut self,
request: impl tonic::IntoRequest<super::NativePreparationRecoverRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationRecoverResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/NativePreparationRecover");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "NativePreparationRecover"));
self.inner.unary(req, path, codec).await
}
pub async fn native_preparation_locate(
&mut self,
request: impl tonic::IntoRequest<super::NativePreparationLocateRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationLocateResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/NativePreparationLocate");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "NativePreparationLocate"));
self.inner.unary(req, path, codec).await
}
pub async fn delete(
&mut self,
request: impl tonic::IntoRequest<super::DeleteRequest>,
@@ -3455,6 +3754,26 @@ pub mod node_service_server {
&self,
request: tonic::Request<super::CompareAndUpdateFileRequest>,
) -> std::result::Result<tonic::Response<super::CompareAndUpdateFileResponse>, tonic::Status>;
async fn native_preparation_begin(
&self,
request: tonic::Request<super::NativePreparationBeginRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationBeginResponse>, tonic::Status>;
async fn native_preparation_finalize(
&self,
request: tonic::Request<super::NativePreparationFinalizeRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationFinalizeResponse>, tonic::Status>;
async fn native_preparation_probe(
&self,
request: tonic::Request<super::NativePreparationProbeRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationProbeResponse>, tonic::Status>;
async fn native_preparation_recover(
&self,
request: tonic::Request<super::NativePreparationRecoverRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationRecoverResponse>, tonic::Status>;
async fn native_preparation_locate(
&self,
request: tonic::Request<super::NativePreparationLocateRequest>,
) -> std::result::Result<tonic::Response<super::NativePreparationLocateResponse>, tonic::Status>;
async fn delete(
&self,
request: tonic::Request<super::DeleteRequest>,
@@ -4181,6 +4500,146 @@ pub mod node_service_server {
};
Box::pin(fut)
}
"/node_service.NodeService/NativePreparationBegin" => {
#[allow(non_camel_case_types)]
struct NativePreparationBeginSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::NativePreparationBeginRequest> for NativePreparationBeginSvc<T> {
type Response = super::NativePreparationBeginResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::NativePreparationBeginRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::native_preparation_begin(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = NativePreparationBeginSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/NativePreparationFinalize" => {
#[allow(non_camel_case_types)]
struct NativePreparationFinalizeSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::NativePreparationFinalizeRequest> for NativePreparationFinalizeSvc<T> {
type Response = super::NativePreparationFinalizeResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::NativePreparationFinalizeRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::native_preparation_finalize(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = NativePreparationFinalizeSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/NativePreparationProbe" => {
#[allow(non_camel_case_types)]
struct NativePreparationProbeSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::NativePreparationProbeRequest> for NativePreparationProbeSvc<T> {
type Response = super::NativePreparationProbeResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::NativePreparationProbeRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::native_preparation_probe(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = NativePreparationProbeSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/NativePreparationRecover" => {
#[allow(non_camel_case_types)]
struct NativePreparationRecoverSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::NativePreparationRecoverRequest> for NativePreparationRecoverSvc<T> {
type Response = super::NativePreparationRecoverResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::NativePreparationRecoverRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::native_preparation_recover(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = NativePreparationRecoverSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/NativePreparationLocate" => {
#[allow(non_camel_case_types)]
struct NativePreparationLocateSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::NativePreparationLocateRequest> for NativePreparationLocateSvc<T> {
type Response = super::NativePreparationLocateResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::NativePreparationLocateRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::native_preparation_locate(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = NativePreparationLocateSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/Delete" => {
#[allow(non_camel_case_types)]
struct DeleteSvc<T: NodeService>(pub Arc<T>);
@@ -7542,3 +8001,331 @@ pub mod tier_mutation_control_service_server {
const NAME: &'static str = SERVICE_NAME;
}
}
/// Generated client implementations.
pub mod capture_raft_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::http::Uri;
use tonic::codegen::*;
#[derive(Debug, Clone)]
pub struct CaptureRaftServiceClient<T> {
inner: tonic::client::Grpc<T>,
}
impl CaptureRaftServiceClient<tonic::transport::Channel> {
/// Attempt to create a new client by connecting to a given endpoint.
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
where
D: TryInto<tonic::transport::Endpoint>,
D::Error: Into<StdError>,
{
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
Ok(Self::new(conn))
}
}
impl<T> CaptureRaftServiceClient<T>
where
T: tonic::client::GrpcService<tonic::body::Body>,
T::Error: Into<StdError>,
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
{
pub fn new(inner: T) -> Self {
let inner = tonic::client::Grpc::new(inner);
Self { inner }
}
pub fn with_origin(inner: T, origin: Uri) -> Self {
let inner = tonic::client::Grpc::with_origin(inner, origin);
Self { inner }
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> CaptureRaftServiceClient<InterceptedService<T, F>>
where
F: tonic::service::Interceptor,
T::ResponseBody: Default,
T: tonic::codegen::Service<
http::Request<tonic::body::Body>,
Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
>,
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
Into<StdError> + std::marker::Send + std::marker::Sync,
{
CaptureRaftServiceClient::new(InterceptedService::new(inner, interceptor))
}
/// Compress requests with the given encoding.
///
/// This requires the server to support it otherwise it might respond with an
/// error.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.send_compressed(encoding);
self
}
/// Enable decompressing responses.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.accept_compressed(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_decoding_message_size(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_encoding_message_size(limit);
self
}
pub async fn append_entries(
&mut self,
request: impl tonic::IntoRequest<super::CaptureRaftRequest>,
) -> std::result::Result<tonic::Response<super::CaptureRaftResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.CaptureRaftService/AppendEntries");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.CaptureRaftService", "AppendEntries"));
self.inner.unary(req, path, codec).await
}
pub async fn vote(
&mut self,
request: impl tonic::IntoRequest<super::CaptureRaftRequest>,
) -> std::result::Result<tonic::Response<super::CaptureRaftResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.CaptureRaftService/Vote");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.CaptureRaftService", "Vote"));
self.inner.unary(req, path, codec).await
}
pub async fn install_snapshot(
&mut self,
request: impl tonic::IntoRequest<super::CaptureRaftRequest>,
) -> std::result::Result<tonic::Response<super::CaptureRaftResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.CaptureRaftService/InstallSnapshot");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.CaptureRaftService", "InstallSnapshot"));
self.inner.unary(req, path, codec).await
}
}
}
/// Generated server implementations.
pub mod capture_raft_service_server {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::*;
/// Generated trait containing gRPC methods that should be implemented for use with CaptureRaftServiceServer.
#[async_trait]
pub trait CaptureRaftService: std::marker::Send + std::marker::Sync + 'static {
async fn append_entries(
&self,
request: tonic::Request<super::CaptureRaftRequest>,
) -> std::result::Result<tonic::Response<super::CaptureRaftResponse>, tonic::Status>;
async fn vote(
&self,
request: tonic::Request<super::CaptureRaftRequest>,
) -> std::result::Result<tonic::Response<super::CaptureRaftResponse>, tonic::Status>;
async fn install_snapshot(
&self,
request: tonic::Request<super::CaptureRaftRequest>,
) -> std::result::Result<tonic::Response<super::CaptureRaftResponse>, tonic::Status>;
}
#[derive(Debug)]
pub struct CaptureRaftServiceServer<T> {
inner: Arc<T>,
accept_compression_encodings: EnabledCompressionEncodings,
send_compression_encodings: EnabledCompressionEncodings,
max_decoding_message_size: Option<usize>,
max_encoding_message_size: Option<usize>,
}
impl<T> CaptureRaftServiceServer<T> {
pub fn new(inner: T) -> Self {
Self::from_arc(Arc::new(inner))
}
pub fn from_arc(inner: Arc<T>) -> Self {
Self {
inner,
accept_compression_encodings: Default::default(),
send_compression_encodings: Default::default(),
max_decoding_message_size: None,
max_encoding_message_size: None,
}
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
where
F: tonic::service::Interceptor,
{
InterceptedService::new(Self::new(inner), interceptor)
}
/// Enable decompressing requests with the given encoding.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.accept_compression_encodings.enable(encoding);
self
}
/// Compress responses with the given encoding, if the client supports it.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.send_compression_encodings.enable(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.max_decoding_message_size = Some(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.max_encoding_message_size = Some(limit);
self
}
}
impl<T, B> tonic::codegen::Service<http::Request<B>> for CaptureRaftServiceServer<T>
where
T: CaptureRaftService,
B: Body + std::marker::Send + 'static,
B::Error: Into<StdError> + std::marker::Send + 'static,
{
type Response = http::Response<tonic::body::Body>;
type Error = std::convert::Infallible;
type Future = BoxFuture<Self::Response, Self::Error>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, req: http::Request<B>) -> Self::Future {
match req.uri().path() {
"/node_service.CaptureRaftService/AppendEntries" => {
#[allow(non_camel_case_types)]
struct AppendEntriesSvc<T: CaptureRaftService>(pub Arc<T>);
impl<T: CaptureRaftService> tonic::server::UnaryService<super::CaptureRaftRequest> for AppendEntriesSvc<T> {
type Response = super::CaptureRaftResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::CaptureRaftRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as CaptureRaftService>::append_entries(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = AppendEntriesSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.CaptureRaftService/Vote" => {
#[allow(non_camel_case_types)]
struct VoteSvc<T: CaptureRaftService>(pub Arc<T>);
impl<T: CaptureRaftService> tonic::server::UnaryService<super::CaptureRaftRequest> for VoteSvc<T> {
type Response = super::CaptureRaftResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::CaptureRaftRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as CaptureRaftService>::vote(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = VoteSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.CaptureRaftService/InstallSnapshot" => {
#[allow(non_camel_case_types)]
struct InstallSnapshotSvc<T: CaptureRaftService>(pub Arc<T>);
impl<T: CaptureRaftService> tonic::server::UnaryService<super::CaptureRaftRequest> for InstallSnapshotSvc<T> {
type Response = super::CaptureRaftResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::CaptureRaftRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as CaptureRaftService>::install_snapshot(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = InstallSnapshotSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
_ => Box::pin(async move {
let mut response = http::Response::new(tonic::body::Body::default());
let headers = response.headers_mut();
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
Ok(response)
}),
}
}
}
impl<T> Clone for CaptureRaftServiceServer<T> {
fn clone(&self) -> Self {
let inner = self.inner.clone();
Self {
inner,
accept_compression_encodings: self.accept_compression_encodings,
send_compression_encodings: self.send_compression_encodings,
max_decoding_message_size: self.max_decoding_message_size,
max_encoding_message_size: self.max_encoding_message_size,
}
}
}
/// Generated gRPC service name
pub const SERVICE_NAME: &str = "node_service.CaptureRaftService";
impl<T> tonic::server::NamedService for CaptureRaftServiceServer<T> {
const NAME: &'static str = SERVICE_NAME;
}
}
+690 -1
View File
@@ -1099,6 +1099,198 @@ pub fn canonical_write_all_request_body(
Ok(body.finish())
}
pub const NATIVE_PREPARATION_METADATA_MAX_BYTES: usize = 1024 * 1024;
pub fn canonical_native_preparation_begin_request_body(
request: &proto_gen::node_service::NativePreparationBeginRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-begin-request-v1\0");
body.push_str(&request.disk)?;
body.push_bytes(&request.disk_id)?;
body.push_bytes(&request.store_id)?;
body.push_bytes(&request.preparation_id)?;
body.push_str(&request.bucket)?;
body.push_str(&request.key)?;
body.push_bytes(&request.bucket_incarnation_id)?;
Ok(body.finish())
}
pub fn canonical_native_preparation_finalize_request_body(
request: &proto_gen::node_service::NativePreparationFinalizeRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-finalize-request-v1\0");
body.push_str(&request.disk)?;
body.push_bytes(&request.disk_id)?;
body.push_bytes(&request.store_id)?;
body.push_bytes(&request.preparation_id)?;
body.push_str(&request.bucket)?;
body.push_str(&request.key)?;
body.push_bytes(&request.bucket_incarnation_id)?;
body.push_bytes(&request.object_digest)?;
body.push_u64(request.object_length);
body.push_bytes(&request.version_id)?;
body.push_bytes(&request.metadata)?;
Ok(body.finish())
}
pub fn canonical_native_preparation_probe_request_body(
request: &proto_gen::node_service::NativePreparationProbeRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-probe-request-v1\0");
body.push_str(&request.disk)?;
body.push_bytes(&request.disk_id)?;
body.push_bytes(&request.store_id)?;
body.push_bytes(&request.preparation_id)?;
body.push_str(&request.bucket)?;
body.push_str(&request.key)?;
body.push_bytes(&request.bucket_incarnation_id)?;
Ok(body.finish())
}
pub fn canonical_native_preparation_recover_request_body(
request: &proto_gen::node_service::NativePreparationRecoverRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-recover-request-v1\0");
body.push_str(&request.disk)?;
body.push_bytes(&request.disk_id)?;
body.push_bytes(&request.store_id)?;
body.push_bytes(&request.preparation_id)?;
body.push_str(&request.bucket)?;
body.push_str(&request.key)?;
body.push_bytes(&request.bucket_incarnation_id)?;
Ok(body.finish())
}
pub fn canonical_native_preparation_locate_request_body(
request: &proto_gen::node_service::NativePreparationLocateRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-locate-request-v1\0");
body.push_str(&request.disk)?;
body.push_bytes(&request.disk_id)?;
body.push_bytes(&request.store_id)?;
body.push_bytes(&request.preparation_id)?;
Ok(body.finish())
}
fn push_native_preparation_error(
body: &mut CanonicalBodyBuilder,
error: Option<&proto_gen::node_service::Error>,
) -> Result<(), std::num::TryFromIntError> {
body.push_bool(error.is_some());
if let Some(error) = error {
body.push_u32(error.code);
body.push_str(&error.error_info)?;
}
Ok(())
}
fn push_native_preparation_receipt(
body: &mut CanonicalBodyBuilder,
receipt: Option<&proto_gen::node_service::NativePreparationReceipt>,
) -> Result<(), std::num::TryFromIntError> {
body.push_bool(receipt.is_some());
let Some(receipt) = receipt else {
return Ok(());
};
body.push_bytes(&receipt.disk_id)?;
body.push_bytes(&receipt.store_id)?;
body.push_bytes(&receipt.deployment_id)?;
body.push_bytes(&receipt.preparation_id)?;
body.push_bytes(&receipt.version_id)?;
body.push_str(&receipt.bucket)?;
body.push_str(&receipt.key)?;
body.push_bytes(&receipt.bucket_incarnation_id)?;
body.push_bytes(&receipt.object_digest)?;
body.push_u64(receipt.object_length);
body.push_bytes(&receipt.set_digest)?;
body.push_bytes(&receipt.distribution_digest)?;
body.push_bytes(&receipt.metadata_digest)?;
body.push_bytes(&receipt.shard_digest)?;
body.push_u64(receipt.shard_length);
body.push_u32(receipt.data_blocks);
body.push_u32(receipt.parity_blocks);
body.push_u32(receipt.shard_index);
Ok(())
}
pub fn canonical_native_preparation_begin_response_body(
request: &proto_gen::node_service::NativePreparationBeginRequest,
response: &proto_gen::node_service::NativePreparationBeginResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let request_body = canonical_native_preparation_begin_request_body(request)?;
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-begin-response-v1\0");
body.push_bytes(&request_body)?;
body.push_bool(response.success);
push_native_preparation_error(&mut body, response.error.as_ref())?;
Ok(body.finish())
}
pub fn canonical_native_preparation_finalize_response_body(
request: &proto_gen::node_service::NativePreparationFinalizeRequest,
response: &proto_gen::node_service::NativePreparationFinalizeResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let request_body = canonical_native_preparation_finalize_request_body(request)?;
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-finalize-response-v1\0");
body.push_bytes(&request_body)?;
body.push_bool(response.success);
push_native_preparation_receipt(&mut body, response.receipt.as_ref())?;
push_native_preparation_error(&mut body, response.error.as_ref())?;
Ok(body.finish())
}
pub fn canonical_native_preparation_probe_response_body(
request: &proto_gen::node_service::NativePreparationProbeRequest,
response: &proto_gen::node_service::NativePreparationProbeResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let request_body = canonical_native_preparation_probe_request_body(request)?;
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-probe-response-v1\0");
body.push_bytes(&request_body)?;
body.push_bool(response.success);
body.push_bool(response.exists);
body.push_bool(response.prepared);
body.push_bytes(&response.metadata)?;
push_native_preparation_error(&mut body, response.error.as_ref())?;
body.push_bytes(&response.version_id)?;
body.push_bytes(&response.object_digest)?;
body.push_u64(response.object_length);
body.push_bytes(&response.set_digest)?;
body.push_bytes(&response.distribution_digest)?;
Ok(body.finish())
}
pub fn canonical_native_preparation_recover_response_body(
request: &proto_gen::node_service::NativePreparationRecoverRequest,
response: &proto_gen::node_service::NativePreparationRecoverResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let request_body = canonical_native_preparation_recover_request_body(request)?;
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-recover-response-v1\0");
body.push_bytes(&request_body)?;
body.push_bool(response.success);
push_native_preparation_receipt(&mut body, response.receipt.as_ref())?;
body.push_bytes(&response.metadata)?;
push_native_preparation_error(&mut body, response.error.as_ref())?;
Ok(body.finish())
}
pub fn canonical_native_preparation_locate_response_body(
request: &proto_gen::node_service::NativePreparationLocateRequest,
response: &proto_gen::node_service::NativePreparationLocateResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let request_body = canonical_native_preparation_locate_request_body(request)?;
let mut body = CanonicalBodyBuilder::new(b"rustfs-native-preparation-locate-response-v1\0");
body.push_bytes(&request_body)?;
body.push_bool(response.success);
body.push_bool(response.found);
body.push_str(&response.bucket)?;
body.push_str(&response.key)?;
body.push_bytes(&response.bucket_incarnation_id)?;
body.push_bytes(&response.metadata)?;
body.push_bool(response.prepared);
push_native_preparation_error(&mut body, response.error.as_ref())?;
Ok(body.finish())
}
pub fn canonical_compare_and_update_file_request_body(
request: &proto_gen::node_service::CompareAndUpdateFileRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
@@ -1269,7 +1461,10 @@ pub fn canonical_make_volumes_request_body(
mod disk_mutation_canonical_tests {
use super::proto_gen::node_service::{
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest,
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, NativePreparationBeginRequest,
NativePreparationFinalizeRequest, NativePreparationFinalizeResponse, NativePreparationLocateRequest,
NativePreparationLocateResponse, NativePreparationProbeRequest, NativePreparationProbeResponse, NativePreparationReceipt,
NativePreparationRecoverRequest, NativePreparationRecoverResponse, PreparePartTransactionRequest, RenameDataRequest,
RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
};
@@ -1318,6 +1513,500 @@ mod disk_mutation_canonical_tests {
assert_all_distinct(&bodies);
}
#[test]
fn native_preparation_begin_body_binds_every_field() {
let baseline = NativePreparationBeginRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
};
let mut bodies = vec![canonical_native_preparation_begin_request_body(&baseline).unwrap()];
for mutate in [
|r: &mut NativePreparationBeginRequest| r.disk = "node-b".into(),
|r: &mut NativePreparationBeginRequest| r.disk_id = vec![5; 16].into(),
|r: &mut NativePreparationBeginRequest| r.store_id = vec![5; 16].into(),
|r: &mut NativePreparationBeginRequest| r.preparation_id = vec![5; 16].into(),
|r: &mut NativePreparationBeginRequest| r.bucket = "other".into(),
|r: &mut NativePreparationBeginRequest| r.key = "other.jpg".into(),
|r: &mut NativePreparationBeginRequest| r.bucket_incarnation_id = vec![5; 16].into(),
] {
let mut request = baseline.clone();
mutate(&mut request);
bodies.push(canonical_native_preparation_begin_request_body(&request).unwrap());
}
assert_all_distinct(&bodies);
}
#[test]
fn native_preparation_finalize_body_binds_every_field() {
let baseline = NativePreparationFinalizeRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
object_digest: vec![5; 32].into(),
object_length: 1234,
version_id: vec![6; 16].into(),
metadata: vec![7, 8, 9].into(),
};
let mut bodies = vec![canonical_native_preparation_finalize_request_body(&baseline).unwrap()];
for mutate in [
|r: &mut NativePreparationFinalizeRequest| r.disk = "node-b".into(),
|r: &mut NativePreparationFinalizeRequest| r.disk_id = vec![9; 16].into(),
|r: &mut NativePreparationFinalizeRequest| r.store_id = vec![9; 16].into(),
|r: &mut NativePreparationFinalizeRequest| r.preparation_id = vec![9; 16].into(),
|r: &mut NativePreparationFinalizeRequest| r.bucket = "other".into(),
|r: &mut NativePreparationFinalizeRequest| r.key = "other.jpg".into(),
|r: &mut NativePreparationFinalizeRequest| r.bucket_incarnation_id = vec![9; 16].into(),
|r: &mut NativePreparationFinalizeRequest| r.object_digest = vec![9; 32].into(),
|r: &mut NativePreparationFinalizeRequest| r.object_length = 4321,
|r: &mut NativePreparationFinalizeRequest| r.version_id = vec![9; 16].into(),
|r: &mut NativePreparationFinalizeRequest| r.metadata = vec![7, 8, 10].into(),
] {
let mut request = baseline.clone();
mutate(&mut request);
bodies.push(canonical_native_preparation_finalize_request_body(&request).unwrap());
}
assert_all_distinct(&bodies);
}
#[test]
fn native_preparation_probe_body_binds_every_field() {
let baseline = NativePreparationProbeRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
};
let mut bodies = vec![canonical_native_preparation_probe_request_body(&baseline).unwrap()];
for mutate in [
|r: &mut NativePreparationProbeRequest| r.disk = "node-b".into(),
|r: &mut NativePreparationProbeRequest| r.disk_id = vec![5; 16].into(),
|r: &mut NativePreparationProbeRequest| r.store_id = vec![5; 16].into(),
|r: &mut NativePreparationProbeRequest| r.preparation_id = vec![5; 16].into(),
|r: &mut NativePreparationProbeRequest| r.bucket = "other".into(),
|r: &mut NativePreparationProbeRequest| r.key = "other.jpg".into(),
|r: &mut NativePreparationProbeRequest| r.bucket_incarnation_id = vec![5; 16].into(),
] {
let mut request = baseline.clone();
mutate(&mut request);
bodies.push(canonical_native_preparation_probe_request_body(&request).unwrap());
}
assert_all_distinct(&bodies);
}
#[test]
fn native_preparation_recover_body_binds_every_field() {
let baseline = NativePreparationRecoverRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
};
let mut bodies = vec![canonical_native_preparation_recover_request_body(&baseline).unwrap()];
for mutate in [
|r: &mut NativePreparationRecoverRequest| r.disk = "node-b".into(),
|r: &mut NativePreparationRecoverRequest| r.disk_id = vec![5; 16].into(),
|r: &mut NativePreparationRecoverRequest| r.store_id = vec![5; 16].into(),
|r: &mut NativePreparationRecoverRequest| r.preparation_id = vec![5; 16].into(),
|r: &mut NativePreparationRecoverRequest| r.bucket = "other".into(),
|r: &mut NativePreparationRecoverRequest| r.key = "other.jpg".into(),
|r: &mut NativePreparationRecoverRequest| r.bucket_incarnation_id = vec![5; 16].into(),
] {
let mut request = baseline.clone();
mutate(&mut request);
bodies.push(canonical_native_preparation_recover_request_body(&request).unwrap());
}
assert_all_distinct(&bodies);
}
fn native_preparation_receipt() -> NativePreparationReceipt {
NativePreparationReceipt {
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
deployment_id: vec![3; 16].into(),
preparation_id: vec![4; 16].into(),
version_id: vec![5; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![6; 16].into(),
object_digest: vec![7; 32].into(),
object_length: 1234,
set_digest: vec![8; 32].into(),
distribution_digest: vec![9; 32].into(),
metadata_digest: vec![10; 32].into(),
shard_digest: vec![11; 32].into(),
shard_length: 512,
data_blocks: 4,
parity_blocks: 2,
shard_index: 3,
}
}
#[test]
fn native_preparation_begin_response_binds_request_status_and_error() {
use super::proto_gen::node_service::{Error, NativePreparationBeginResponse};
let request = NativePreparationBeginRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
};
let response = NativePreparationBeginResponse {
success: true,
error: None,
response_proof: vec![1; 32].into(),
};
let baseline = canonical_native_preparation_begin_response_body(&request, &response).unwrap();
let failed = NativePreparationBeginResponse {
success: false,
error: Some(Error {
code: 17,
error_info: "disk unavailable".into(),
}),
..response.clone()
};
assert_ne!(baseline, canonical_native_preparation_begin_response_body(&request, &failed).unwrap());
let mut changed_request = request.clone();
changed_request.disk_id = vec![9; 16].into();
assert_ne!(
baseline,
canonical_native_preparation_begin_response_body(&changed_request, &response).unwrap()
);
let mut proof_only = response.clone();
proof_only.response_proof = vec![2; 32].into();
assert_eq!(baseline, canonical_native_preparation_begin_response_body(&request, &proof_only).unwrap());
}
#[test]
fn native_preparation_finalize_response_binds_request_status_error_and_receipt() {
use super::proto_gen::node_service::Error;
let request = NativePreparationFinalizeRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
object_digest: vec![5; 32].into(),
object_length: 1234,
version_id: vec![6; 16].into(),
metadata: vec![7, 8].into(),
};
let response = NativePreparationFinalizeResponse {
success: true,
receipt: Some(native_preparation_receipt()),
error: None,
response_proof: vec![1; 32].into(),
};
let baseline = canonical_native_preparation_finalize_response_body(&request, &response).unwrap();
let mut bodies = vec![baseline.clone()];
for mutate in [
|r: &mut NativePreparationReceipt| r.disk_id = vec![12; 16].into(),
|r: &mut NativePreparationReceipt| r.store_id = vec![12; 16].into(),
|r: &mut NativePreparationReceipt| r.deployment_id = vec![12; 16].into(),
|r: &mut NativePreparationReceipt| r.preparation_id = vec![12; 16].into(),
|r: &mut NativePreparationReceipt| r.version_id = vec![12; 16].into(),
|r: &mut NativePreparationReceipt| r.bucket = "other".into(),
|r: &mut NativePreparationReceipt| r.key = "other.jpg".into(),
|r: &mut NativePreparationReceipt| r.bucket_incarnation_id = vec![12; 16].into(),
|r: &mut NativePreparationReceipt| r.object_digest = vec![12; 32].into(),
|r: &mut NativePreparationReceipt| r.object_length = 4321,
|r: &mut NativePreparationReceipt| r.set_digest = vec![12; 32].into(),
|r: &mut NativePreparationReceipt| r.distribution_digest = vec![12; 32].into(),
|r: &mut NativePreparationReceipt| r.metadata_digest = vec![12; 32].into(),
|r: &mut NativePreparationReceipt| r.shard_digest = vec![12; 32].into(),
|r: &mut NativePreparationReceipt| r.shard_length = 1024,
|r: &mut NativePreparationReceipt| r.data_blocks = 8,
|r: &mut NativePreparationReceipt| r.parity_blocks = 4,
|r: &mut NativePreparationReceipt| r.shard_index = 5,
] {
let mut changed = response.clone();
mutate(changed.receipt.as_mut().unwrap());
bodies.push(canonical_native_preparation_finalize_response_body(&request, &changed).unwrap());
}
assert_all_distinct(&bodies);
let failed = NativePreparationFinalizeResponse {
success: false,
error: Some(Error {
code: 19,
error_info: "finalize failed".into(),
}),
..response.clone()
};
assert_ne!(baseline, canonical_native_preparation_finalize_response_body(&request, &failed).unwrap());
let mut changed_request = request.clone();
changed_request.metadata = vec![7, 9].into();
assert_ne!(
baseline,
canonical_native_preparation_finalize_response_body(&changed_request, &response).unwrap()
);
let mut proof_only = response.clone();
proof_only.response_proof = vec![2; 32].into();
assert_eq!(
baseline,
canonical_native_preparation_finalize_response_body(&request, &proof_only).unwrap()
);
}
#[test]
fn native_preparation_probe_response_binds_state_receipt_metadata_and_error() {
use super::proto_gen::node_service::Error;
let request = NativePreparationProbeRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
};
let response = NativePreparationProbeResponse {
success: true,
exists: true,
prepared: true,
metadata: vec![1, 2, 3].into(),
error: None,
response_proof: vec![1; 32].into(),
version_id: vec![4; 16].into(),
object_digest: vec![5; 32].into(),
object_length: 4096,
set_digest: vec![6; 32].into(),
distribution_digest: vec![7; 32].into(),
};
let baseline = canonical_native_preparation_probe_response_body(&request, &response).unwrap();
let mut bodies = vec![baseline.clone()];
for (exists, prepared) in [(false, true), (true, false), (false, false)] {
let changed = NativePreparationProbeResponse {
exists,
prepared,
..response.clone()
};
bodies.push(canonical_native_preparation_probe_response_body(&request, &changed).unwrap());
}
let metadata_changed = NativePreparationProbeResponse {
metadata: vec![1, 2, 4].into(),
..response.clone()
};
bodies.push(canonical_native_preparation_probe_response_body(&request, &metadata_changed).unwrap());
for changed in [
NativePreparationProbeResponse {
version_id: vec![8; 16].into(),
..response.clone()
},
NativePreparationProbeResponse {
object_digest: vec![8; 32].into(),
..response.clone()
},
NativePreparationProbeResponse {
object_length: 8192,
..response.clone()
},
NativePreparationProbeResponse {
set_digest: vec![8; 32].into(),
..response.clone()
},
NativePreparationProbeResponse {
distribution_digest: vec![8; 32].into(),
..response.clone()
},
] {
bodies.push(canonical_native_preparation_probe_response_body(&request, &changed).unwrap());
}
assert_all_distinct(&bodies);
let failed = NativePreparationProbeResponse {
success: false,
error: Some(Error {
code: 21,
error_info: "probe failed".into(),
}),
..response.clone()
};
assert_ne!(baseline, canonical_native_preparation_probe_response_body(&request, &failed).unwrap());
let mut changed_request = request.clone();
changed_request.preparation_id = vec![9; 16].into();
assert_ne!(
baseline,
canonical_native_preparation_probe_response_body(&changed_request, &response).unwrap()
);
}
#[test]
fn native_preparation_recover_response_binds_request_receipt_metadata_and_error() {
use super::proto_gen::node_service::Error;
let request = NativePreparationRecoverRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
};
let response = NativePreparationRecoverResponse {
success: true,
receipt: Some(native_preparation_receipt()),
metadata: vec![1, 2, 3].into(),
error: None,
response_proof: vec![1; 32].into(),
};
let baseline = canonical_native_preparation_recover_response_body(&request, &response).unwrap();
let metadata_changed = NativePreparationRecoverResponse {
metadata: vec![1, 2, 4].into(),
..response.clone()
};
assert_ne!(
baseline,
canonical_native_preparation_recover_response_body(&request, &metadata_changed).unwrap()
);
let failed = NativePreparationRecoverResponse {
success: false,
error: Some(Error {
code: 23,
error_info: "recover failed".into(),
}),
..response.clone()
};
assert_ne!(baseline, canonical_native_preparation_recover_response_body(&request, &failed).unwrap());
let mut changed_request = request.clone();
changed_request.key = "other.jpg".into();
assert_ne!(
baseline,
canonical_native_preparation_recover_response_body(&changed_request, &response).unwrap()
);
let mut proof_only = response.clone();
proof_only.response_proof = vec![2; 32].into();
assert_eq!(
baseline,
canonical_native_preparation_recover_response_body(&request, &proof_only).unwrap()
);
}
#[test]
fn native_preparation_locate_bodies_bind_route_and_description() {
use super::proto_gen::node_service::Error;
let request = NativePreparationLocateRequest {
disk: "node-a".into(),
disk_id: vec![1; 16].into(),
store_id: vec![2; 16].into(),
preparation_id: vec![3; 16].into(),
};
let response = NativePreparationLocateResponse {
success: true,
found: true,
bucket: "photos".into(),
key: "image.jpg".into(),
bucket_incarnation_id: vec![4; 16].into(),
metadata: vec![5, 6, 7].into(),
prepared: true,
error: None,
response_proof: vec![1; 32].into(),
};
let baseline = canonical_native_preparation_locate_response_body(&request, &response).unwrap();
let mut bodies = vec![baseline.clone()];
for changed in [
NativePreparationLocateResponse {
success: false,
..response.clone()
},
NativePreparationLocateResponse {
found: false,
..response.clone()
},
NativePreparationLocateResponse {
bucket: "other".into(),
..response.clone()
},
NativePreparationLocateResponse {
key: "other.jpg".into(),
..response.clone()
},
NativePreparationLocateResponse {
bucket_incarnation_id: vec![8; 16].into(),
..response.clone()
},
NativePreparationLocateResponse {
metadata: vec![5, 6, 8].into(),
..response.clone()
},
NativePreparationLocateResponse {
prepared: false,
..response.clone()
},
NativePreparationLocateResponse {
error: Some(Error {
code: 25,
error_info: "locate failed".into(),
}),
..response.clone()
},
] {
bodies.push(canonical_native_preparation_locate_response_body(&request, &changed).unwrap());
}
assert_all_distinct(&bodies);
let mut changed_request = request.clone();
changed_request.store_id = vec![9; 16].into();
assert_ne!(
baseline,
canonical_native_preparation_locate_response_body(&changed_request, &response).unwrap()
);
let mut proof_only = response.clone();
proof_only.response_proof = vec![2; 32].into();
assert_eq!(
baseline,
canonical_native_preparation_locate_response_body(&request, &proof_only).unwrap()
);
let mut request_bodies = vec![canonical_native_preparation_locate_request_body(&request).unwrap()];
for mutate in [
|r: &mut NativePreparationLocateRequest| r.disk = "node-b".into(),
|r: &mut NativePreparationLocateRequest| r.disk_id = vec![9; 16].into(),
|r: &mut NativePreparationLocateRequest| r.store_id = vec![9; 16].into(),
|r: &mut NativePreparationLocateRequest| r.preparation_id = vec![9; 16].into(),
] {
let mut changed = request.clone();
mutate(&mut changed);
request_bodies.push(canonical_native_preparation_locate_request_body(&changed).unwrap());
}
assert_all_distinct(&request_bodies);
}
#[test]
fn rename_data_canonical_body_is_injective_across_field_boundaries() {
// Length prefixes must prevent moving bytes between adjacent fields from colliding.
+140
View File
@@ -142,6 +142,132 @@ message CompareAndUpdateFileResponse {
optional Error error = 3;
}
// Native preparation carries only bounded object metadata over the control RPC.
// Shard bytes use the authenticated data stream and remain private until finalized.
message NativePreparationReceipt {
bytes disk_id = 1; // 16-byte UUID.
bytes store_id = 2; // 16-byte UUID.
bytes deployment_id = 3; // 16-byte UUID.
bytes preparation_id = 4; // 16-byte UUID.
bytes version_id = 5; // 16-byte UUID.
string bucket = 6;
string key = 7;
bytes bucket_incarnation_id = 8; // 16-byte UUID.
bytes object_digest = 9; // 32-byte SHA-256.
uint64 object_length = 10;
bytes set_digest = 11;
bytes distribution_digest = 12;
bytes metadata_digest = 13;
bytes shard_digest = 14;
uint64 shard_length = 15;
uint32 data_blocks = 16;
uint32 parity_blocks = 17;
uint32 shard_index = 18;
reserved 19, 20;
}
message NativePreparationBeginRequest {
string disk = 1;
bytes disk_id = 2; // 16-byte UUID.
bytes store_id = 3; // 16-byte UUID.
bytes preparation_id = 4; // 16-byte UUID.
string bucket = 5;
string key = 6;
bytes bucket_incarnation_id = 7; // 16-byte UUID.
}
message NativePreparationBeginResponse {
bool success = 1;
optional Error error = 2;
bytes response_proof = 3;
}
message NativePreparationFinalizeRequest {
string disk = 1;
bytes disk_id = 2; // 16-byte UUID.
bytes store_id = 3; // 16-byte UUID.
bytes preparation_id = 4; // 16-byte UUID.
string bucket = 5;
string key = 6;
bytes bucket_incarnation_id = 7; // 16-byte UUID.
bytes object_digest = 8; // 32-byte SHA-256.
uint64 object_length = 9;
bytes version_id = 10; // 16-byte UUID.
// Exact serialized FileMeta; maximum 1 MiB. Peers reject larger values.
bytes metadata = 11;
}
message NativePreparationFinalizeResponse {
bool success = 1;
NativePreparationReceipt receipt = 2;
optional Error error = 3;
bytes response_proof = 4;
}
message NativePreparationProbeRequest {
string disk = 1;
bytes disk_id = 2; // 16-byte UUID.
bytes store_id = 3; // 16-byte UUID.
bytes preparation_id = 4; // 16-byte UUID.
string bucket = 5;
string key = 6;
bytes bucket_incarnation_id = 7; // 16-byte UUID.
}
message NativePreparationProbeResponse {
bool success = 1;
bool exists = 2;
bool prepared = 3;
reserved 4;
// Exact serialized FileMeta when it exists and can be read; maximum 1 MiB.
bytes metadata = 5;
optional Error error = 6;
bytes response_proof = 7;
bytes version_id = 8; // 16-byte UUID; observation only, not a durability receipt.
bytes object_digest = 9; // 32-byte SHA-256.
uint64 object_length = 10;
bytes set_digest = 11;
bytes distribution_digest = 12;
}
message NativePreparationRecoverRequest {
string disk = 1;
bytes disk_id = 2; // 16-byte UUID.
bytes store_id = 3; // 16-byte UUID.
bytes preparation_id = 4; // 16-byte UUID.
string bucket = 5;
string key = 6;
bytes bucket_incarnation_id = 7; // 16-byte UUID.
}
message NativePreparationRecoverResponse {
bool success = 1;
NativePreparationReceipt receipt = 2;
// Exact persisted FileMeta; maximum 1 MiB. Peers reject larger values.
bytes metadata = 3;
optional Error error = 4;
bytes response_proof = 5;
}
message NativePreparationLocateRequest {
string disk = 1;
bytes disk_id = 2; // 16-byte UUID.
bytes store_id = 3; // 16-byte UUID.
bytes preparation_id = 4; // 16-byte UUID.
}
message NativePreparationLocateResponse {
bool success = 1;
bool found = 2;
string bucket = 3;
string key = 4;
bytes bucket_incarnation_id = 5; // 16-byte UUID.
bytes metadata = 6; // Exact serialized FileMeta; maximum 1 MiB.
bool prepared = 7;
optional Error error = 8;
bytes response_proof = 9;
}
message DeleteRequest {
string disk = 1; // indicate which one in the disks
string volume = 2;
@@ -1216,6 +1342,11 @@ service NodeService {
rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {}; // auth-policy: read-only
rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {}; // auth-policy: body-bound
rpc CompareAndUpdateFile(CompareAndUpdateFileRequest) returns (CompareAndUpdateFileResponse) {}; // auth-policy: body-bound
rpc NativePreparationBegin(NativePreparationBeginRequest) returns (NativePreparationBeginResponse) {}; // auth-policy: body-bound
rpc NativePreparationFinalize(NativePreparationFinalizeRequest) returns (NativePreparationFinalizeResponse) {}; // auth-policy: body-bound
rpc NativePreparationProbe(NativePreparationProbeRequest) returns (NativePreparationProbeResponse) {}; // auth-policy: body-bound
rpc NativePreparationRecover(NativePreparationRecoverRequest) returns (NativePreparationRecoverResponse) {}; // auth-policy: body-bound
rpc NativePreparationLocate(NativePreparationLocateRequest) returns (NativePreparationLocateResponse) {}; // auth-policy: body-bound
rpc Delete(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound
rpc DeleteAtIncarnation(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound
rpc AcquireSnapshotLease(SnapshotLeaseRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
@@ -1331,3 +1462,12 @@ service TierMutationControlService {
rpc CommitTierMutation(TierMutationCommitRequest) returns (TierMutationControlResponse) {};
rpc AbortTierMutation(TierMutationAbortRequest) returns (TierMutationControlResponse) {};
}
// Capture authority uses a bounded, versioned canonical body, not protobuf re-encoding.
message CaptureRaftRequest { bytes body = 1; }
message CaptureRaftResponse { bytes body = 1; bytes response_proof = 2; }
service CaptureRaftService {
rpc AppendEntries(CaptureRaftRequest) returns (CaptureRaftResponse);
rpc Vote(CaptureRaftRequest) returns (CaptureRaftResponse);
rpc InstallSnapshot(CaptureRaftRequest) returns (CaptureRaftResponse);
}
+2
View File
@@ -8,6 +8,8 @@ collection:
## Operations
- [Native capture Raft runtime](operations/native-capture.md): explicit creation, recovery, shared reader, and shutdown.
- [Architecture guard troubleshooting](operations/architecture-guard-troubleshooting.md) — diagnose repository guard failures.
- [Logging governance](operations/logging-governance.md) — broad event audits, migrations, and guardrail changes.
+7
View File
@@ -32,7 +32,13 @@ Required headings and strings in these files are asserted by `scripts/check_arch
| Document | Use this when |
|---|---|
| [native-raft-transport.md](native-raft-transport.md) | mounting authenticated TCP capture Raft RPCs or configuring bounded transport |
| [native-raft-owner.md](native-raft-owner.md) | starting real capture Raft lifecycle or wiring native decision submission and a shared reader |
| [native-raft-state-machine.md](native-raft-state-machine.md) | adapting native capture application state to OpenRaft or receiving durable snapshots |
| [capture-raft-log-storage.md](capture-raft-log-storage.md) | adapting native capture persistence to OpenRaft log traits or changing checkpoint-authorized prefix purge |
| [capture-checkpoint-contract.md](capture-checkpoint-contract.md) | changing Capture checkpoint capture, provenance, installation, recovery, or log cleanup |
| [native-source-reader.md](native-source-reader.md) | exposing a live native capture writer through a same-UID local read-only source interface |
| [native-object-preparation.md](native-object-preparation.md) | changing private native object preparation, durable shard receipts, retry placement, or recovery before capture publication |
| [local-descriptor-cache.md](local-descriptor-cache.md) | changing descriptor cache keys/open modes or invalidation after local mutations |
| [erasure-coding.md](erasure-coding.md) | changing anything under `crates/ecstore/src/erasure/`, `crates/filemeta/`, `crates/ecstore/src/set_disk/`, storage-class or layout code, or any decode, quorum, or heal boundary (normative spec) |
| [placement-repair-invariants.md](placement-repair-invariants.md) | changing anything that resolves an object to a pool, set, or disk, or that admits scanner or heal work |
@@ -50,6 +56,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
| [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md) | deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary |
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
| [plugin-delegation.md](plugin-delegation.md) | changing plugin delegation registration, revocation, broker identity, or the scope of attempt credentials |
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another |
| [tier-stats-contract.md](tier-stats-contract.md) | changing what `GET /rustfs/admin/v3/tier-stats` returns, adding a tier accounting source, or wiring a metric to a remote tier request |
@@ -34,6 +34,65 @@ caller validation and require isolated staging. Rows remain opaque to the
framing decoder. A framing digest detects byte corruption; it does not establish
semantic consistency, historical truth, or authority to install a checkpoint.
## Local checkpoint implementation
`NativeLogWriter::checkpoint_local` in
`crates/capture-authority/src/native_checkpoint.rs` accepts resource limits and
captures its own store. It has no artifact, path, or caller-supplied trust input.
The private staging owner retains the original read transaction and store;
decoded rows must match that transaction byte for byte before publication.
This establishes local provenance without treating a valid snapshot digest as
remote authorization.
One admission guard limits checkpoint work per writer. Capturing and validating
the snapshot allow later application writes to proceed. Publication takes the
existing writer ordering guard and retains it in the blocking worker through
the durable transaction. It replaces four immutable baseline tables and their
versioned metadata atomically, advancing local storage to schema 7. Live
application tables, identity, vote, committed state, and later log entries stay
in the same database. Old schema readers reject this explicit storage upgrade;
CSNP v1 and COPR v1 retain their encodings.
`checkpoint_baseline::validate_recovery` validates baseline relationships with
disk-backed indexes, then applies the continuous suffix into a temporary redb
store using the existing `apply_created` decision logic. It compares recovered
rows and membership with the live state. Original negative results remain
canonical across this boundary. Temporary databases use a bounded file backend
and fixed cache; snapshot records stream through bounded buffers. The recorded
budget is the default for recovery. If later state requires more temporary
space, `Store::open_with_checkpoint_limits` supplies a larger explicit budget;
resource exhaustion is not reported as corrupt history.
`NativeLogWriter::purge_checkpoint` checks the selected generation and complete
cut, validates recovery from the durable baseline, then checks the selection
again in the cleanup transaction. That transaction removes covered log entries
and application responses together with a durable full-ID purge marker. A stale
receipt cannot authorize cleanup of another generation. Recovery accepts the
purged boundary when the committed marker equals the exact baseline cut, and
requires all retained log entries to be contiguous after the purge marker.
```mermaid
sequenceDiagram
participant W as Native writer
participant C as Local capture worker
participant D as Durable redb store
W->>C: Admit one checkpoint
C->>D: Acquire owned read transaction at cut
D-->>C: Header and four application tables
W->>D: Continue append and apply after cut
C->>C: Stream, decode, compare, validate on disk
C->>W: Acquire writer ordering
C->>D: Atomically publish baseline and schema 7
D-->>W: Generation and full cut receipt
W->>D: Re-read baseline and validate suffix recovery
W->>D: Atomically purge covered logs/results and record boundary
```
These local operations are prerequisites for Raft log compaction. They do not
provide a remote snapshot installation endpoint, authenticated Raft transport,
quorum commitment, or S3 publication. Those boundaries must be connected to the
real authority owner before source events may claim published-object authority.
## Canonical operation provenance
Checkpoint-capable storage must use an explicitly versioned operation envelope
@@ -0,0 +1,29 @@
# Capture Raft log storage
**Use this when:** adapting the native capture writer to OpenRaft log persistence or changing checkpoint-authorized log prefix cleanup.
`crates/capture-authority/src/raft_log_storage.rs` implements `NativeRaftLogStore::new(NativeLogWriter)` and the actual OpenRaft 0.9.25 `RaftLogStorage<CaptureTypeConfig>` and `RaftLogReader` traits. Clones share the original Store and write ordering; no adapter opens another redb database. The crate enables the existing dependency's `storage-v2` feature without changing its version.
## Durable operations
Vote, append, truncate, committed-marker writes and purge run through the existing native blocking writer. Append supplies its real `LogFlushed` callback to that worker and reports success only after the original durable commit returns. Failure reports an error rather than success, including the existing indeterminate-commit write fence. Cancellation of an awaiting append does not release the worker's ordering guard or callback before storage work finishes. Lock order remains writer ordering followed by Store; no Store mutex is held across an await.
`get_log_state` reads the durable purged marker and actual retained log tail, checks the stored tail agrees, and reports the purged ID as the tail when no log remains. Applied progress is not used as a substitute. Committed-marker writes reuse the existing exact-identity and monotonicity checks. `save_committed(None)` cannot clear an existing marker.
`get_log_reader` and direct reader calls delegate to the existing `NativeLogReader`. The adapter translates native failures to typed OpenRaft storage errors; it does not reinterpret application responses or manufacture consensus results.
## Authorized prefix purge
A purge target requires the currently selected durable local checkpoint and validated baseline/suffix recovery evidence. It must lie at or before the checkpoint cut and match the complete retained LogId, including term and leader. Missing checkpoints, uncovered targets, mismatched identity and write-disabled handles reject before deletion.
The original writer ordering and Store ownership cover validation through commit. The transaction rechecks the selected checkpoint and exact target, deletes only LOGS and APPLY_RESULTS through that target, and advances the existing purged marker atomically. Retained logs above the target and all EVENTS, OPERATIONS and object/version state remain intact. Repeating the exact current purged ID succeeds without moving the marker; older or conflicting requests cannot rewind it or claim an identity no longer retained.
The existing `NativeLogWriter::purge_checkpoint` receipt API still checks its exact generation and checkpoint cut, then invokes the same operation at that cut. The on-disk purged marker and checkpoint encodings are unchanged: existing recovery already validates the remaining prefix from purged+1 through the baseline cut. Partial purge therefore needs no migration.
## Verification and owner boundary
The `raft_log_storage` tests invoke actual storage traits against real redb. `RaftLogStorageExt::blocking_append` is the public OpenRaft helper that constructs its own private `LogFlushed`, invokes the adapter and waits for the callback. Tests do not forge callbacks or substitute direct writer append for trait acceptance. Reopen and the existing disk-fault backend exercise flush persistence, cancellation, failed commits, partial purge, identity rejection and retained suffix continuation. Existing native checkpoint and source reader tests verify compatibility and event retention.
This adapter is the production authority owner's log-storage prerequisite. It does not implement `RaftStateMachine`, snapshot receive/install, the actual Raft owner, network transport or S3 publication. Trusted test preparation uses existing native apply/checkpoint calls without claiming real consensus. S06 remains blocked and R12 remains frozen; this slice does not complete U05/U06.
Rollback can remove adapter consumers while retaining the existing native writer, checkpoint and disk formats. Prefixes already durably purged remain recoverable from their selected checkpoint and retained suffix; rollback must not discard that baseline.
@@ -0,0 +1,74 @@
# Native Object Preparation
**Use this when:** changing private native object preparation, durable shard receipts, retry placement, or recovery before capture publication.
## Authority boundary
`ECStore::prepare_native_object` in `crates/ecstore/src/store/native_prepare.rs`
consumes actual reader bytes and returns `NativeQuorumPreparation` only after a
write quorum of distinct local disks has durably prepared matching object facts.
Its private receipt collection prevents callers from constructing a quorum out
of unchecked values. A preparation proves neither capture consensus nor public
object visibility. Its version identifier is private metadata identity, not an
assertion that an S3 version is readable. Preparation identifiers and capture
decision identifiers remain separate.
```mermaid
flowchart LR
R[Actual object reader] --> P[Probe private placement]
P --> E[Existing erasure and bitrot encoder]
E --> D[Local disk intent and shard barriers]
D --> Q[Distinct matching write-quorum receipts]
Q --> O[Private preparation result]
```
## Encoding and placement
Small local candidates retain the bounded inline preparation path. Larger or
unknown-length candidates use the ordinary streaming erasure/bitrot encoder to
write private part files, with a fixed-size SHA-256 read buffer rather than
buffering the object. Unsupported owners fail explicitly; preparation never
falls back to a visible public PUT.
`SetDisks::prepare_native_bytes` in
`crates/ecstore/src/set_disk/ops/native_prepare.rs` uses the existing write layout
and encoder. Verification uses the codec indicated by the decoded metadata,
including its shard width and checksum compatibility mode.
New candidates use the existing pool/set selection. Recovery searches actual
sets for the preparation UUID, then verifies the retained physical disk set and
distribution rather than recomputing placement from the current key hash.
Existing private intents fix placement for retries. Absence is established only against a healthy disk identity;
unreadable state in another pool prevents assigning a new pool. Conflicting
identities are errors, even if another subset could form a quorum.
## Persistence and cancellation
`LocalDisk::prepare_native` in
`crates/ecstore/src/disk/local/native_prepare.rs` owns the metadata, object, and
private preparation lock chain through the blocking I/O and durability barriers.
Detached disk writes retain the parent object owner after early quorum return or
caller cancellation. An unobserved response does not authorize rollback.
Version 2 intents retain the original raw metadata so an intent-only interruption
can be repaired without reserializing metadata maps. Version 3 streaming intents
also retain the ordered physical disk UUID set and data directory UUID. The exact
FileInfo supplies the part manifest; each file's bitrot data and the aggregate
shard digest/length are verified before the record and prepared marker barriers.
Part files and their directory are synced before a disk can attest preparation. Existing version 1 intents
keep their original canonical representation and marker. A missing shard with an
existing prepared marker is corruption, not a successful preparation.
## Recovery
Recovery acquires fresh lifecycle and namespace owners, verifies persisted disk
facts, decodes the actual shards, and checks the complete object length and SHA-256.
It requires the write quorum and consistent identity, codec, modification time,
part manifest and block geometry. It decodes each retained part into a streaming
digest sink, so recovery does not allocate a buffer proportional to object size. Retry preserves original metadata bytes and rejects changed
content. Fewer receipts or unknown I/O outcomes leave durable facts available for
recovery without granting publication authority.
The actual filesystem tests are in
`crates/ecstore/src/store/native_prepare_tests.rs` and
`crates/ecstore/src/store/native_prepare_multi_pool_tests.rs`; local barrier and
format compatibility tests remain beside the LocalDisk implementation.
+37
View File
@@ -0,0 +1,37 @@
# Native Raft owner
**Use this when:** starting the capture authority's actual Raft lifecycle, submitting native decisions, or wiring a shared source reader.
`crates/capture-authority/src/raft_owner.rs` provides `NativeRaftOwner::start(writer, node_id, config, limits, network)`. It validates OpenRaft configuration, snapshot limits, writable native application state, capture binding and the runtime ID against durable META identity before calling real OpenRaft 0.9.25 `Raft::new`. The log and state-machine adapters receive clones of the same `NativeLogWriter`; neither opens another live database. Start never initializes membership, clears storage, or generates a replacement identity.
## Formation, writes and progress
`bootstrap` is an explicit first-cluster operation. Its member map must contain the durable local node, and every `CaptureNode` address must pass the existing validator. It then invokes actual `Raft::initialize`. The caller is responsible for consistent initial membership across nodes; start on a recovered database must not bootstrap it again. OpenRaft's concrete initialization failure is preserved.
`client_write` submits the existing typed `CaptureCommand` to actual Raft. Its `ClientWriteResponse<CaptureTypeConfig>` contains the real consensus entry LogId and native application response. The first decision identity inside an idempotent Created response can precede the outer Raft retry entry; successful retries do not create another authoritative event.
`NativeRaftOwnerError` retains concrete OpenRaft write, initialization and Fatal errors, including ForwardToLeader. `metrics` returns the existing watch receiver with leader, applied and fatal state. Lack of a leader, loss of majority, caller timeout or transport failure is not a committed-success result. An interrupted request can have an unknown outcome: preserve the same operation identity when retrying. The owner does not manufacture a success response from local staging or direct apply.
## Reader and shutdown ownership
On Linux and macOS, `bind_reader` binds the accepted `SourceReaderServer` with this same writer. It returns the server and shutdown signal; the caller owns its serve task. Stop that reader and await its drain before releasing storage. The owner does not claim to drain a server task it does not own.
`shutdown` stops admission to new owner bootstrap/write/reader-binding calls, then invokes and awaits real `Raft::shutdown`. OpenRaft 0.9.25 joins its core and tick, but does not join its state-machine worker or detached snapshot builders. A shared lifetime lease in the actual native state machine, its builder clones and the owner-created log storage/replication readers therefore extends shutdown until their final real drop, after their writer/Store fields have been released. A cancelled asynchronous replication read also carries the lease into its real blocking worker until that worker releases Store; external writer clones are not attached to this owner lease. The completion state survives cancelled shutdown waiters; notification is registered before checking it. Calls racing the admission boundary can already be in Raft; only their actual returned result proves success. Ordinary async cancellation preserves the previously accepted storage adapters' blocking commit ownership. Shutdown also waits for owner-created replication readers. An injected network that never finishes its RPC can therefore delay shutdown; this owner supplies no default transport deadline. A production factory must honor the OpenRaft RPC timeout contract. Dropping the owner is not an async shutdown barrier; a cancelled shutdown waiter should call shutdown again and await completion.
`raft` exposes the existing handle for explicit transport registration. Raw handle clones and the original writer are trusted internal caller-owned references; this owner type does not seal every bypass. Do not manually append or apply through writer clones while Raft is running. Before reopening the same redb file, shut down Raft and the reader and release external writer, Raft and factory references. Shutdown does not revoke independent read handles or delete EVENTS.
## Transport boundary and verification
A `RaftNetworkFactory<CaptureTypeConfig>` is mandatory. This module has no default or production network implementation and does not automatically connect to existing RustFS RPC. Authentication, cluster/member/target binding, signed bodies, replay protection, snapshot transfer framing and transport timeouts remain subsequent production work. Factory implementers must use the accepted snapshot receive path and its bounded handles.
`crates/capture-authority/src/raft_owner_tests.rs` uses real Raft instances and real redb log/state-machine adapters. Its direct RPC factory is only a test transport: it forwards append, vote and snapshot calls to actual peer Raft handles. Tests cover explicit bootstrap, restart without initialize, retry identity, runtime identity rejection, live shared reader, shutdown, three-node majority, leader failover and snapshot catch-up through actual Raft RPC. Readiness comes from metrics and authoritative events, rather than a fixed sleep.
These tests establish the internal owner and consensus path under that test transport. They do not prove production network trust or wire native capture into S3 PUT, Copy or multipart publication. S06 remains blocked, R12 remains frozen, and production S3 automatic capture remains a later integration task. Removing this lifecycle adapter does not change the accepted native storage/snapshot formats; retain the selected durable baseline after prefix purge and follow its existing empty-baseline rollback restrictions.
## Exact Created publication authorization
`NativeRaftOwner::authorize_created(expected, expected_decision)` reads the canonical operation, its successful first result, event, version, capture binding, native provenance and applied/committed boundaries under one ordered Store read transaction. Supply the complete `DecideCreated` and the successful result's first full decision LogId. An idempotent retry's outer log index does not replace that identity. Rejected or unapplied operations cannot mint authority.
The returned `CommittedCreatedAuthorization` has private fields, read-only accessors, no public constructor and no deserialization. It binds the durable cluster, operation, object incarnation/key, expected head, preparation/version/digest/length, binding and first decision. Its node identity identifies the current issuing owner, not a requirement to publish on the original leader. Recovery reads retained provenance and returns the same decision facts after actual Raft recovery.
This local type is not a remote signature or proof of physical object durability. Publication must separately validate the real durable preparation and current bucket incarnation, namespace and head before making bytes visible. It does not revoke historical authorization when a later head is created, and it provides no external command endpoint. Streaming preparation and S3 publication consume this boundary separately.
@@ -0,0 +1,33 @@
# Native Raft state machine and snapshots
**Use this when:** adapting native capture application state to OpenRaft, receiving snapshots, or investigating snapshot recovery and shared log ownership.
`crates/capture-authority/src/raft_state_machine.rs` provides `NativeRaftStateMachine::new(NativeLogWriter, CheckpointLimits)`. It implements the actual OpenRaft 0.9.25 state-machine and snapshot-builder traits. Pair it with `NativeRaftLogStore` using clones of the same writer. Apply uses the existing durable native application, membership and typed responses; no second live database is opened.
## Selected baseline and transport
Nonempty builds publish through `NativeLogWriter::checkpoint_local`. Empty builds publish a verified empty baseline under the same write ordering. Both selections include complete snapshot metadata in the same durable transaction. Export reads the selected baseline header, four baseline tables and metadata from one redb read transaction. It never substitutes live tables from a later applied cut. `get_current_snapshot` reconstructs that selection after restart.
The received ID is persisted verbatim. Locally generated IDs include cluster, local node and durable generation, so different nodes do not reuse a generation-only ID. Old nonempty baselines without full metadata derive this local ID; newly published metadata has a required marker, so missing metadata cannot silently fall back. Recovery verifies generation, complete applied LogId and membership against the selected header. A subsequent local checkpoint replaces the incoming metadata atomically.
`crates/capture-authority/src/native_snapshot_data.rs` implements bounded synchronous and asynchronous read/write/seek. Receive capacity is the configured frame limit plus the 56-byte capability manifest. Writes and seeks check arithmetic and capacity before changing state or allocating; allocation is fallible. The transport contains the existing capability manifest followed by the existing four-section frame. No private publisher types or new snapshot codec are involved.
## Validate and install
`crates/capture-authority/src/native_snapshot_install.rs` decodes into an isolated bounded staging database. It checks capability integrity, frame resource limits, membership, provenance and four-table relationships. Incoming metadata must match the full decoded applied ID and membership. The durable destination cluster and capture binding must match; local node identity and vote are preserved.
Publication acquires writer ordering then Store ownership. One Immediate durable transaction selects the baseline and metadata, replaces live tables and the disposable event index, and updates applied, membership, committed, tail and purged markers. A compatible complete cut retains its continuous suffix, including committed entries not yet applied. Conflicting uncommitted suffixes are discarded; applied history and conflicting committed history cannot be rolled back. A cut beyond the local tail becomes the durable purged boundary. Empty baselines recover by replaying from index zero through the same bounded disk-backed scratch path used for nonempty baselines, including operator recovery-limit overrides.
Cancellation before final publication rejects; a blocking commit retains ownership and the received handle until it finishes even if its async caller is dropped. Indeterminate commits retain the existing write fence. Reopen observes the complete old or complete new state. Snapshot EVENTS remain authoritative and are neither consumed nor deleted by publisher acknowledgements.
## Receive and purge ordering
OpenRaft can enqueue state-machine install and then run log purge before the separate state-machine worker begins installation. `begin_receiving_snapshot` therefore attaches a shared receive guard to the bounded handle. Install requires a handle belonging to that writer; callers of a future full-snapshot transport must use this trait entry point, rather than constructing an unrelated buffer.
When a purge lacks baseline coverage and a receive is pending, the log adapter waits for receive completion without holding writer ordering. Successful install, rejection and receive cancellation release the guard and wake waiters. Multiple handles retain independent ownership. Waiting has an absolute 30-second bound; a malformed or stalled receive cannot indefinitely block an uncovered purge. An uncovered purge without a pending receive still rejects immediately. Install itself atomically establishes the covered purged boundary, making the subsequent same-cut purge idempotent.
## Compatibility and verification
Existing nonempty schema-7 baselines remain readable. Empty baselines use the existing encoding with an absent applied cut, which older binaries reject. Before rolling back to such a binary, publish a compatible nonempty checkpoint or retain the new version. Do not discard the selected baseline after logs have been purged. The complete metadata keys are additive; incoming snapshot IDs must not be regenerated during recovery.
`raft_state_machine_tests.rs` and `native_snapshot_install_tests.rs` invoke actual traits on two real redb stores, check restart and continuation, bound receives, reject corrupt identity, preserve committed suffixes, and inject cancellation/power cuts around final commits. Deterministic pending-receive tests exercise the trait install/purge race and cancellation wakeup. These are local storage tests: they do not establish network consensus, a live Raft owner, or production S3 automatic capture. S06 remains blocked and R12 remains frozen.
@@ -0,0 +1,35 @@
# Native Raft transport
**Use this when:** mounting authenticated capture Raft RPCs, configuring its network factory, or investigating message, replay and lifecycle limits.
`crates/ecstore/src/cluster/rpc/capture_raft.rs` implements the production `CaptureRaftNetworkFactory` and `CaptureRaftRpc`. Capture authority remains independent of ECStore. `NativeRaftOwner::identity` returns the identity read from durable META before that owner started; the server derives its cluster/node identity from this accessor. Factory/server constructors validate explicit fixed members and addresses. A network client accepts the target node metadata only when it exactly matches that member map. Vote/append/snapshot leader identity must match the authenticated body's source node; the destination and cluster must match the actual owner.
## Mount and authentication
`CaptureRaftRpc::into_service` returns a bounded generated tonic service for mounting beside existing `NodeService/Ping`. Authentication occurs inside each Raft handler exactly once. Future RustFS mounting must not also apply the old HTTP `check_auth` interceptor, which would consume the same nonce twice. This slice supplies no standalone production daemon or startup integration.
`crates/ecstore/src/cluster/rpc/client.rs` provides the narrow capture client helper. It reuses existing channel/TLS/cache configuration, signature interceptor and `AuthenticatedChannel`. Every exchange performs an actual authenticated Ping challenge/proof handshake before dispatch. The existing peer epoch/capability cache is learned from that response, never prefilled. Missing or revoked authenticated capability rejects the call.
All three Raft methods require the existing v3 boot-scoped header before using `verify_tonic_rpc_signature_with_bootstrap(false)` and strict canonical body-digest verification. This does not depend on compatibility switches enabling strictness, and has no v1/unsigned fallback. The existing nonce cache rejects replay and the server's process epoch rejects old-boot signatures. The server returns the existing boot epoch/capability proof headers with successful and actual Raft error replies, preserving the authenticated channel's peer state. Rejections count through the existing internode authentication metric; no new credential protocol or registry is introduced. This uses RustFS's shared internal RPC credential trust, not individual peer certificates or plugin-runner service mTLS.
## Wire contract and errors
`crates/protos/src/node.proto` adds only `CaptureRaftService` and opaque canonical request/response bodies plus a response proof. The version-1 JSON envelope strictly binds cluster, source, target, method, fresh request nonce and the complete typed payload. Input byte bounds precede parsing. Typed re-encoding must exactly match the received bytes, rejecting unknown fields, duplicate/noncanonical fields and trailing data. Nested payloads use `RawValue` and the same typed serialization on both ends; they never pass through a string-sorted generic JSON map. Existing command and explicit capture membership validation still applies.
The response proof reuses the existing internode HMAC helper over a domain-separated, length-delimited complete original request and complete result. It therefore includes the original nonce, payload/digest semantics, cluster/target and method. Changed results, old replies and cross-request reuse fail verification before returning a Raft result.
Success values retain actual OpenRaft responses. Remote stopped, panicked, storage-I/O, defensive-storage and snapshot-mismatch classes remain typed `RPCError::RemoteError`; transport rejection and timeout cannot become success. Original storage error strings and backtraces are never sent. A bounded local serde projection preserves safe I/O subject/verb when available; if the projection exceeds its 16-KiB budget, the wire marks that operation unavailable and the local reconstruction uses a redacted generic I/O context. Defensive errors retain their typed subject/violation without backtrace.
## Limits and lifecycle
All members must use consistent message limits; heterogeneous receiver limits are not negotiated. Defaults are 16 MiB request body, 1 MiB response body, 300 append entries, 3 MiB snapshot chunk and 16 concurrent admitted calls per factory/server. Connect timeout is three seconds and configured request timeout five seconds. Constructors reject invalid/overflowing limits and Raft batch/chunk configurations that exceed them. Snapshot capacity accounts for worst-case four JSON bytes per raw byte plus header allowance. The append count matches OpenRaft's default 300; no entries are silently truncated. If a multi-entry encoded request exceeds the byte limit, `PayloadTooLarge` gives OpenRaft a smaller count hint so it can retry a complete prefix. A single entry that cannot fit is explicitly rejected.
Bounded encoding uses checked lengths and fallible growth. Each exchange's timeout includes admission, encoding, handshake and RPC, capped by `RPCOption::soft_ttl()` before its hard deadline. The server holds its admission permit in an owned task through the actual Raft future and signed reply; cancelling an HTTP waiter does not release that permit or abort the queued Raft operation. Such cancellation has an unknown remote outcome, not a success guarantee.
Keep a `CaptureRaftRpc` clone to manage mounting. Its `shutdown` stops new admission and waits for all actual accepted tasks by acquiring the full permit set; cancellation can be followed by another shutdown wait. Stop/await the tonic listener and then shut down the actual Raft owner and release external references before reopening the same redb. This does not replace the owner's accepted SM/builder/replication drain or source-reader shutdown contract.
## Verification and remaining work
`crates/ecstore/src/cluster/rpc/capture_raft_tests.rs` uses three real TCP listeners, the production factory and service, three actual Raft instances and real redb. Its minimal Ping route calls production signing/boot-proof functions. A frame-level test fault wrapper alters only delivery/proofs; it does not replace the Raft handler or manually apply decisions. Tests cover majority, failover, real snapshot RPC catch-up, identity/authentication/replay, remote errors, size/deadline/shutdown, byte-limit batch splitting and cancelled HTTP admission. The worker pause test proves ownership of accepted service work before actual Raft dispatch, not a simulated redb commit.
Production RustFS startup mounting and S3 automatic capture are later slices. No existing service/authentication behavior or capture storage format is replaced. The generated protobuf remains additive; older peers without this service cannot join this capture Raft transport through a downgrade fallback.
+29
View File
@@ -0,0 +1,29 @@
# Native source reader
**Use this when:** exposing a live `NativeLogWriter` to a local publisher without reopening its redb database or granting event mutation authority.
## Ownership and API
`crates/capture-authority/src/source_reader.rs` owns the public native read protocol, server and client. Within an IO-enabled Tokio runtime, the actual authority owner calls `SourceReaderServer::bind(path, writer.clone())`, keeps the returned `Shutdown`, and spawns `server.serve()`. `Shutdown::shutdown()` stops new admissions; await the serve task to drain accepted reads and report socket cleanup errors. `NativeLogWriter::durable_identity()` reads and validates the identity persisted in META.
The supplied writer shares its existing Store and blocking worker boundary. The service reads durable identity and `created_events_page` under the same Store lock. It never opens another live database, rewrites an event, records an ACK, or advances a durable source cursor. Event delivery position does not remove EVENTS, including after checkpoint publication. All services bound from clones of one writer share the same admission budget.
`SourceReaderClient::new(path)` validates the private socket location. Its only operation is `page(expected_cluster, cursor, limit)`. Returned `ReadPageResponse` values contain the durable `NodeIdentity`, complete native `SourceEvent` records and `EventPage::next`. They describe local committed events and are not Controller receipts or consumer ACKs. The consumer retains its previous cursor until its independent durable acceptance policy permits advancement.
## Trust and transport
Support is limited to Linux and macOS. Other targets do not expose this module. The immediate directory must belong to the effective UID and forbid every group/other permission; a symlink directory is rejected. The service creates a socket with mode 0600 and refuses every existing path, including stale sockets and symlinks. The caller owns the pathname lifecycle exclusively while the service or its cleanup is active; do not externally unlink or rebind that path during this interval. Operators must investigate a stale path themselves; the service never takes it over.
Both server and client check connected peer credentials against the current effective UID. The client also validates directory/socket ownership and permissions before connecting, then checks their device/inode identity after connection. Effective UID changes while the service is running are unsupported. This is a shared same-UID trust domain: it does not isolate mutually hostile processes running as that same user. Do not put the socket in a directory exposed to another user or use it as a network service.
Frames contain a four-byte big-endian unsigned JSON byte length followed by compact UTF-8 JSON. `ReadPage`, `ReadPageReply` and all nested native types use strict field decoding; wire version 1 is required. Request size is at most 4096 bytes, response size at most 8 MiB, and page size at most `MAX_EVENT_PAGE`. The server rejects wrong clusters before reading event pages. Existing stored cursor/event checks remain authoritative. Native storage and checkpoint codecs are unchanged.
At most four connections per writer are admitted, including blocking reads from cancelled services. Request reads and response writes each have a five-second deadline; the client bounds the complete exchange at five seconds. Serialization uses a byte-limited writer. The blocking store worker retains its admission permit through reading and serialization even if a client disconnects or its caller is cancelled. Storage work is drained rather than forcibly interrupted. Graceful shutdown can therefore wait for an existing local storage operation beyond the network deadline. Dropping the serve future cancels network tasks, while already started blocking reads retain admission until they finish. A replacement service on the same writer cannot bypass those permits.
Cleanup checks the original socket inode and private directory before unlinking. Detected replacements are preserved and reported as cleanup failures. This check and unlink are separate filesystem operations, so the caller-exclusive pathname lifecycle is required. Cleanup disarms its ownership before its single attempt; dropping an old guard cannot retry against a socket rebound after that attempt. A cancelled serve task makes best-effort cleanup under those same checks. Socket removal is local lifecycle cleanup, not durable source-state deletion.
## Verification and scope
`cargo test -p rustfs-capture-authority source_reader -- --nocapture` exercises actual redb, committed native apply and Unix sockets owned by the test process. It covers live apply visibility, restart continuation, page boundaries, durable identity/cursor rejection, private and occupied paths, transport bounds, active-read shutdown/cancellation, immediate rebinding after cleanup, and checkpoint replay. The cluster-check mutation must fail `SOURCE_READER_CLUSTER` before a foreign page can be returned.
No production authority owner, RustFS main binary or automatic S3 capture is connected by this library change. Existing blocked/frozen source work stays separate. Rollback removes reader consumers and their local socket configuration; stored events, native log bytes and checkpoints remain compatible.
+67
View File
@@ -0,0 +1,67 @@
# Plugin delegation
**Use this when:** changing plugin delegation registration, revocation, broker identity, or the scope of attempt credentials.
## Authority
A delegation is an immutable restriction record owned by an ordinary IAM user. It is neither a permission grant nor a credential. Creating it requires that user's real signed request and `admin:CreatePluginDelegation`; reading and revoking require the corresponding `admin:GetPluginDelegation` and `admin:RevokePluginDelegation` actions. Root, temporary credentials and service accounts cannot own records.
The record binds a deployment, bucket, literal input and output prefixes, and an expiry of at most 24 hours. Wildcards, control characters and IAM variable expressions are rejected. Its random ID is also its immutable revision. There is no update operation.
The Controller is responsible for validating accepted task identity, current lease, epoch, node session and control generation before requesting credentials. RustFS authenticates the configured broker and enforces the delegation and current parent IAM. RustFS does not independently verify a remote Controller lease.
```mermaid
sequenceDiagram
participant U as Original IAM user
participant C as Controller broker
participant R as RustFS
participant P as Plugin
U->>R: Signed delegation registration
R-->>U: Immutable ID and revision
C->>C: Validate task and current lease
C->>R: Signed issue-attempt request
R->>R: Broker mapping + active record + current IAM
R->>R: Persist scoped STS and recheck authority
R-->>C: Short-lived credentials
C-->>P: Attempt credentials
P->>R: Exact version GET / attempt-prefix PUT
R->>R: Current parent IAM intersected with session policy
```
## Persistence and revocation
`crates/iam/src/delegation.rs` owns record validation and immutable revocation tombstones. Records use the existing IAM store under `config/iam/plugin-delegations/v1`. Writes require a successful readback. Revocation is idempotent and durable; it does not delete or overwrite the restriction record.
`ObjectStore::load_iam_config_strict` distinguishes a missing object from an empty object, invalid JSON or failed decryption. Only a genuinely absent tombstone permits an active record. Corrupt or unavailable storage rejects access rather than treating revocation as absent. Existing non-delegation readers keep their compatibility behavior.
Revocation prevents future issuance. It does not promise immediate invalidation of credentials already delivered; those remain subject to their expiration and current parent IAM. Registration, IAM changes and issuance are not one atomic transaction. Issuance rechecks after persistence and deletes the generated STS account if authority changed. Failed persistence or invalidation acknowledgment is reported as unknown issuance, never as safe retry permission.
## Broker profile
`rustfs/src/admin/handlers/plugin_attempt.rs` implements `POST /rustfs/admin/v3/plugin-delegations/{id}/issue-attempt`. The signer must be an ordinary IAM user with `admin:IssuePluginAttempt`, independently listed in `RUSTFS_PLUGIN_BROKERS`:
```json
{"broker-access-key":"deployment-name"}
```
This configuration contains public access keys, not secrets. It is immutable per process. Missing or invalid configuration disables broker issuance. The broker is a trusted Controller identity; ordinary users cannot substitute their own request fields for Controller validation.
The request carries `tenant`, `run_id`, `attempt_id`, `revision`, `deployment`, `input_bucket`, `input_key`, `input_version`, `output_bucket`, `output_prefix` and `ttl_seconds`. IDs must be non-nil. The first profile accepts one exact input version, one bucket, and an output prefix equal to:
```text
<delegation-output-prefix><lowercase-hex-tenant>/<run-uuid>/<attempt-uuid>/
```
The session policy permits only `s3:GetObjectVersion` for that input with an exact `s3:VersionId`, and `s3:PutObject` beneath that attempt prefix. Credential lifetime is at most 900 seconds and cannot exceed delegation expiry. The Controller must additionally cap it to its validated lease and hard deadline.
The original user's current IAM remains the parent permission boundary on every S3 request. The server's signing key only seals the token; it is not the plugin's authorization identity. Broker identity and original user identity are separate.
External policy plugins must be disabled: their authorization path does not enforce this built-in session-policy boundary. Initializing, failed and ready plugin states all reject issuance. Attempt tokens carry a signed, persisted marker: after restart or on another node, marked credentials are accepted only by built-in STS authorization. A denied marked request returns a terminal error, so public bucket policies cannot bypass its session scope. The first profile also rejects parent policies containing conditions, because a broker request cannot reconstruct the original user's IP, MFA or other request context. It does not silently replace that context with the broker's.
## Response and failures
Successful issuance returns `version`, `delegation_id`, `delegation_revision`, `owner_access_key`, `access_key_id`, `secret_access_key`, `session_token` and `expires_at_unix_seconds`, with `Cache-Control: no-store`. The Controller verifies the returned delegation and owner against the accepted caller before delivery. Credentials must never enter logs, metrics or task result manifests.
Missing records and records owned by a different user have the same read response. Revoked or expired records reject access. Storage failures remain errors. Post-persistence failures carry static `error_code`, `issued` and `issuance_unknown` facts without credentials; callers must reconcile an unknown outcome rather than automatically mint another credential.
The route/action inventory is maintained in `rustfs/src/admin/route_policy.rs`. This contract does not imply event capture, physical container cleanup, restart reconciliation or upstream release acceptance.
+53
View File
@@ -0,0 +1,53 @@
# Native capture Raft runtime
Use this when enabling, forming, restarting, or shutting down the native capture Raft runtime in a RustFS process.
Native capture is disabled by default. Enable it with `--native-capture-config /path/to/capture.json` or `RUSTFS_NATIVE_CAPTURE_CONFIG`. The configuration is strict JSON, limited to 64 KiB. Unknown fields are rejected.
```json
{
"identity": {
"cluster_id": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
"node_id": 0
},
"database": "/var/lib/rustfs/capture.redb",
"reader_socket": "/run/rustfs/capture.sock",
"peers": {
"0": {"rpc_uri": "http://127.0.0.1:9000"}
},
"capture_binding": {
"revision": 1,
"binding_id": [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
"target_id": [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]
},
"mode": "create",
"bootstrap": true
}
```
Choose deployment-specific nonzero identifiers. IDs above are examples. Each node has its own database and reader socket, a distinct node ID, and the same cluster identity and capture binding. All peers must use fixed ports. The local peer must match the authority used by the existing RustFS internal RPC Ping handshake. The runtime uses the existing RPC secret; the capture file contains no credentials.
Prepare the database and socket parent directories as the account that runs RustFS. The database parent must exist. The reader socket parent must be owned by that account and have no group or other permission bits. The reader refuses any existing socket pathname; stop its owner before removing a stale path, and do not remove a path belonging to another live process.
```sh
install -d -m 700 /var/lib/rustfs /run/rustfs
chmod 600 /etc/rustfs/capture.json
```
Run these commands with the deployment's service account or provision the directories with that account's ownership. The example configuration's `database` and `reader_socket` must point into these prepared directories. A source reader client connects to `/run/rustfs/capture.sock`, not the HTTP API port. Supply `RUSTFS_RPC_SECRET` through the service manager's protected environment or secret injection before launch; do not put a secret in the JSON, command line, or shell history.
For the example first formation configuration (`mode: create`, `bootstrap: true`), start the real executable on the matching fixed API port:
```sh
rustfs server --address 127.0.0.1:9000 --native-capture-config /etc/rustfs/capture.json /var/lib/rustfs/objects
```
After normal shutdown, retain the database, edit that same file to `mode: recover` and `bootstrap: false`, and run the same command. Do not delete the database to make recovery succeed. On a three-node deployment, use each node's configured local endpoint and object volumes in its launch command; its capture peer authority must match the existing internal RPC identity.
For first formation, use `create` only when the database does not exist. Set `bootstrap` to `true` only on the designated formation node. For three nodes, give every file the same peer map with all three node IDs and their actual RustFS HTTP endpoints; change each file's local node ID and private paths. Other nodes start without bootstrap and receive real Raft membership and replication.
For restart, change `mode` to `recover` and `bootstrap` to `false`. Recovery requires the retained database and its exact identity and binding. A missing database or an identity mismatch is an error. Recovery does not rebuild a lost database or apply membership changes from configuration. Preserve the database when replacing a process. Do not replace an occupied reader socket pathname while its owner is running.
The existing HTTP listener serves authenticated native Raft RPCs. The Unix source reader shares the owner's durable store and supports read-only event-page queries. Shutdown stops admission, waits for actual RPC and owner work, and joins the reader task before releasing storage. Startup failures after the owner is created also require this cleanup.
This process integration does not automatically capture S3 writes. There is no external command-submission endpoint. Production object publication and automatic Created events require the separate object-publication integration. Reader queries prove access to durable native source state; they do not by themselves prove S3 capture.
+2
View File
@@ -223,6 +223,8 @@ hotpath-cpu = [
workspace = true
[dependencies]
rustfs-capture-authority = { workspace = true }
openraft = { workspace = true, features = ["storage-v2"] }
hotpath.workspace = true
# RustFS Internal Crates
rustfs-heal = { workspace = true }
+4
View File
@@ -224,3 +224,7 @@ mod tests {
// The current tests focus on data structures and basic functionality that can be
// tested in isolation without complex dependencies.
}
pub mod plugin_delegation;
pub mod plugin_attempt;
+412
View File
@@ -0,0 +1,412 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Restricted vending for explicitly configured brokers, never AssumeRole chaining.
use super::plugin_delegation::{owner, response, storage_error};
use crate::{
admin::{
router::{AdminOperation, Operation, S3Router},
runtime_sources,
},
auth::check_key_valid_with_context,
server::ADMIN_PREFIX,
};
use http::{Method, StatusCode};
use matchit::Params;
use rustfs_iam::{
delegation::Delegation,
sys::{PLUGIN_ATTEMPT_CREDENTIAL_CLAIM, SESSION_POLICY_NAME},
};
use rustfs_policy::{
auth::get_new_credentials_with_metadata,
policy::{
Args, Policy,
action::{Action, AdminAction, S3Action},
},
};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
sync::OnceLock,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use time::OffsetDateTime;
// Operator configuration contains public signer access keys, not credentials.
// Empty/invalid configuration disables the endpoint. It is immutable per process.
static BROKERS: OnceLock<Option<HashMap<String, String>>> = OnceLock::new();
fn configured_deployment(key: &str) -> Option<&'static str> {
BROKERS
.get_or_init(|| {
let raw = std::env::var("RUSTFS_PLUGIN_BROKERS").ok()?;
let map: HashMap<String, String> = serde_json::from_str(&raw).ok()?;
if map.is_empty()
|| map
.iter()
.any(|(k, v)| k.is_empty() || v.is_empty() || k.len() > 1024 || v.len() > 1024)
{
return None;
}
Some(map)
})
.as_ref()?
.get(key)
.map(String::as_str)
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Attempt {
tenant: String,
run_id: uuid::Uuid,
attempt_id: uuid::Uuid,
revision: String,
deployment: String,
input_bucket: String,
input_key: String,
input_version: String,
output_bucket: String,
output_prefix: String,
ttl_seconds: u64,
}
#[derive(Serialize)]
struct Issued {
version: u32,
delegation_id: String,
delegation_revision: String,
owner_access_key: String,
access_key_id: String,
secret_access_key: String,
session_token: String,
expires_at_unix_seconds: i64,
}
pub fn register_plugin_attempt_route(router: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/issue-attempt"),
AdminOperation(&Issue),
)?;
Ok(())
}
fn now() -> S3Result<u64> {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.map_err(|_| s3_error!(InternalError, "clock unavailable"))
}
fn literal(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 1024
&& !value.contains("${")
&& !value.chars().any(|c| c.is_control() || matches!(c, '*' | '?'))
}
fn session_policy(record: &Delegation, attempt: &Attempt) -> S3Result<Policy> {
let restriction = &record.restrictions;
// Stored records from earlier versions must obey the same literal rules.
restriction.validate(record.created_at_unix_seconds).map_err(storage_error)?;
let exact_output = format!(
"{}{}/{}/{}/",
restriction.output_prefix,
hex_simd::encode_to_string(attempt.tenant.as_bytes(), hex_simd::AsciiCase::Lower),
attempt.run_id,
attempt.attempt_id
);
if attempt.tenant.is_empty()
|| attempt.tenant.len() > 1024
|| attempt.run_id.is_nil()
|| attempt.attempt_id.is_nil()
|| attempt.output_prefix != exact_output
|| attempt.revision != record.revision
|| attempt.deployment != restriction.deployment
|| attempt.input_bucket != restriction.bucket
|| attempt.output_bucket != restriction.bucket
|| !literal(&attempt.input_key)
|| !attempt.input_key.starts_with(&restriction.input_prefix)
|| !literal(&attempt.input_version)
|| attempt.input_version == "null"
|| !literal(&attempt.output_prefix)
|| !attempt.output_prefix.starts_with(&restriction.output_prefix)
|| attempt.output_prefix == restriction.output_prefix
|| !attempt.output_prefix.ends_with('/')
|| !(1..=900).contains(&attempt.ttl_seconds)
{
return Err(s3_error!(AccessDenied, "attempt restriction rejected"));
}
let value = serde_json::json!({"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":["s3:GetObjectVersion"],"Resource":[format!("arn:aws:s3:::{}/{}",attempt.input_bucket,attempt.input_key)],"Condition":{"StringEquals":{"s3:VersionId":attempt.input_version}}},
{"Effect":"Allow","Action":["s3:PutObject"],"Resource":[format!("arn:aws:s3:::{}/{}*",attempt.output_bucket,attempt.output_prefix)]}
]});
// Condition operators borrow their keys while the policy is decoded.
let bytes = serde_json::to_vec(&value).map_err(|_| s3_error!(InternalError, "attempt policy unavailable"))?;
serde_json::from_slice(&bytes).map_err(|_| s3_error!(InternalError, "attempt policy unavailable"))
}
fn issuance_error(code: &'static str, issued: bool, unknown: bool) -> S3Result<S3Response<(StatusCode, Body)>> {
let mut answer = response(&serde_json::json!({"error_code":code,"issued":issued,"issuance_unknown":unknown}))?;
answer.output.0 = StatusCode::SERVICE_UNAVAILABLE;
Ok(answer)
}
async fn require_builtin_authorization() -> S3Result<()> {
rustfs_iam::sys::IamSys::<rustfs_iam::store::object::ObjectStore>::ensure_restricted_credential_vending()
.await
.map_err(|_| s3_error!(AccessDenied, "external policy plugin unsupported for attempt credentials"))
}
struct Issue;
#[async_trait::async_trait]
impl Operation for Issue {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
require_builtin_authorization().await?;
let (iam, broker) = owner(&req, AdminAction::IssuePluginAttemptAction).await?;
let deployment = configured_deployment(&broker).ok_or_else(|| s3_error!(AccessDenied, "broker deployment rejected"))?;
let context = runtime_sources::app_context_from_req(&req)
.ok_or_else(|| s3_error!(InternalError, "application context unavailable"))?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3_error!(InvalidRequest, "attempt body unavailable"))?
.map_err(|_| s3_error!(InvalidRequest, "attempt body unavailable"))?;
let attempt: Attempt =
serde_json::from_slice(&bytes).map_err(|_| s3_error!(InvalidRequest, "invalid attempt request"))?;
if attempt.deployment != deployment {
return Err(s3_error!(AccessDenied, "broker deployment rejected"));
}
let id = params
.get("id")
.ok_or_else(|| s3_error!(InvalidRequest, "delegation id required"))?;
let record = iam
.read_active_plugin_delegation_for_broker(id, now()?)
.await
.map_err(storage_error)?;
let policy = session_policy(&record, &attempt)?;
let (principal, root) = check_key_valid_with_context("", &record.owner_access_key, Some(context.as_ref())).await?;
if root || principal.is_temp() || principal.is_service_account() {
return Err(s3_error!(AccessDenied, "original user required"));
}
// A broker request cannot reconstruct the original user's IP/MFA/tag
// conditions. This first profile rejects conditional parent policies.
let policies = iam
.policy_db_get(&principal.access_key, &principal.groups)
.await
.map_err(|_| s3_error!(InternalError, "parent policy unavailable"))?;
let parent_policy = iam.get_combined_policy(&policies).await;
if policies.is_empty() || parent_policy.statements.iter().any(|s| !s.conditions.is_empty()) {
return Err(s3_error!(AccessDenied, "parent condition context unsupported"));
}
let mut conditions = HashMap::new();
conditions.insert("s3:VersionId".to_owned(), vec![attempt.input_version.clone()]);
if !iam
.is_allowed(&Args {
account: &principal.access_key,
groups: &principal.groups,
action: Action::S3Action(S3Action::GetObjectVersionAction),
bucket: &attempt.input_bucket,
object: &attempt.input_key,
conditions: &conditions,
is_owner: false,
claims: principal.claims_or_empty(),
deny_only: false,
})
.await
{
return Err(s3_error!(AccessDenied, "parent input rejected"));
}
let expiration = now()?
.checked_add(attempt.ttl_seconds)
.ok_or_else(|| s3_error!(InvalidRequest, "invalid expiration"))?
.min(record.restrictions.expires_at_unix_seconds);
let expires_at = OffsetDateTime::from_unix_timestamp(
i64::try_from(expiration).map_err(|_| s3_error!(InvalidRequest, "invalid expiration"))?,
)
.map_err(|_| s3_error!(InvalidRequest, "invalid expiration"))?;
let mut claims = principal.claims.clone().unwrap_or_default();
claims.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), serde_json::Value::Bool(true));
claims.insert("exp".into(), serde_json::json!(expires_at.unix_timestamp()));
claims.insert("parent".into(), serde_json::json!(principal.access_key));
let bytes = serde_json::to_vec(&policy).map_err(|_| s3_error!(InternalError, "attempt policy unavailable"))?;
claims.insert(
SESSION_POLICY_NAME.into(),
serde_json::json!(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&bytes)),
);
let fresh = iam
.read_active_plugin_delegation_for_broker(id, now()?)
.await
.map_err(storage_error)?;
if fresh != record {
return Err(s3_error!(AccessDenied, "attempt authority changed"));
}
owner(&req, AdminAction::IssuePluginAttemptAction).await?;
require_builtin_authorization().await?;
let signing = context
.action_credentials()
.get()
.ok_or_else(|| s3_error!(InternalError, "token signer unavailable"))?;
// The server key seals the token only. Authorization above uses the
// original user's current IAM, and S3 continues its parent intersection.
let mut credential = get_new_credentials_with_metadata(&claims, &signing.secret_key)
.map_err(|_| s3_error!(InternalError, "attempt credential unavailable"))?;
credential.parent_user = principal.access_key.clone();
if iam.set_temp_user(&credential.access_key, &credential, None).await.is_err() {
return issuance_error("PLUGIN_ATTEMPT_ISSUANCE_UNKNOWN", false, true);
}
// No atomic IAM/delegation transaction spans persistence. Recheck and
// invalidate the generated credential if any authority changed.
let valid = async {
require_builtin_authorization().await?;
let after = iam
.read_active_plugin_delegation_for_broker(id, now()?)
.await
.map_err(storage_error)?;
let (current, current_root) =
check_key_valid_with_context("", &record.owner_access_key, Some(context.as_ref())).await?;
let names = iam
.policy_db_get(&current.access_key, &current.groups)
.await
.map_err(|_| s3_error!(InternalError, "parent policy unavailable"))?;
let current_policy = iam.get_combined_policy(&names).await;
if after != record
|| current_root
|| current.is_temp()
|| current.is_service_account()
|| current.secret_key != principal.secret_key
|| names != policies
|| serde_json::to_vec(&current_policy).ok() != serde_json::to_vec(&parent_policy).ok()
{
return Err(s3_error!(AccessDenied, "attempt authority changed"));
}
// Re-evaluate the broker against its real signed request context.
owner(&req, AdminAction::IssuePluginAttemptAction).await?;
Ok::<(), s3s::S3Error>(())
}
.await;
if valid.is_err() {
if iam.delete_temp_account(&credential.access_key, true).await.is_err() {
return issuance_error("PLUGIN_ATTEMPT_INVALIDATION_UNKNOWN", true, true);
}
return issuance_error("PLUGIN_ATTEMPT_POST_ISSUANCE_REJECTED", true, false);
}
response(&Issued {
version: 1,
delegation_id: record.id,
delegation_revision: record.revision,
owner_access_key: record.owner_access_key,
access_key_id: credential.access_key,
secret_access_key: credential.secret_key,
session_token: credential.session_token,
expires_at_unix_seconds: expires_at.unix_timestamp(),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn attempt_policy_keeps_exact_version_and_narrow_output() {
let record = Delegation {
version: 1,
id: "a".repeat(32),
revision: "a".repeat(32),
owner_access_key: "user".into(),
created_at_unix_seconds: 1,
restrictions: rustfs_iam::delegation::Restrictions {
deployment: "runner".into(),
bucket: "photos".into(),
input_prefix: "input/".into(),
output_prefix: "output/".into(),
expires_at_unix_seconds: 1000,
},
};
let run_id = uuid::Uuid::new_v4();
let attempt_id = uuid::Uuid::new_v4();
let output_prefix = format!(
"output/{}/{}/{}/",
hex_simd::encode_to_string(b"tenant", hex_simd::AsciiCase::Lower),
run_id,
attempt_id
);
let mut attempt = Attempt {
tenant: "tenant".into(),
run_id,
attempt_id,
revision: record.revision.clone(),
deployment: "runner".into(),
input_bucket: "photos".into(),
input_key: "input/photo.png".into(),
input_version: "version-1".into(),
output_bucket: "photos".into(),
output_prefix: output_prefix.clone(),
ttl_seconds: 60,
};
let policy = session_policy(&record, &attempt).unwrap();
let scoped_output = format!("{output_prefix}result");
let groups = None;
let claims = HashMap::new();
for (action, object, version, expected) in [
(S3Action::GetObjectVersionAction, "input/photo.png", Some("version-1"), true),
(S3Action::GetObjectVersionAction, "input/photo.png", Some("version-2"), false),
(S3Action::GetObjectVersionAction, "input/photo.png", None, false),
(S3Action::GetObjectAction, "input/photo.png", None, false),
(S3Action::PutObjectAction, scoped_output.as_str(), None, true),
(S3Action::PutObjectAction, "outside/result", None, false),
(S3Action::PutObjectAction, "output/result", None, false),
] {
let conditions = version
.map(|value| HashMap::from([("versionid".into(), vec![value.into()])]))
.unwrap_or_default();
assert_eq!(
policy
.is_allowed(&Args {
account: "user",
groups: &groups,
action: Action::S3Action(action),
bucket: "photos",
object,
conditions: &conditions,
is_owner: false,
claims: &claims,
deny_only: false,
})
.await,
expected,
"attempt policy action={action:?} object={object} version={version:?}"
);
}
attempt.output_prefix = "output/".into();
assert!(session_policy(&record, &attempt).is_err());
attempt.output_prefix = output_prefix;
attempt.input_key = "outside/photo.png".into();
assert!(session_policy(&record, &attempt).is_err());
}
#[tokio::test]
async fn actual_policy_variable_resolution_is_rejected_by_literal_scope() {
let resource = rustfs_policy::policy::resource::Resource::S3("photos/input/${aws:username}.png".into());
let conditions = HashMap::from([("username".into(), vec!["alice".into()])]);
let resolver =
rustfs_policy::policy::variables::VariableResolver::new(rustfs_policy::policy::variables::VariableContext {
username: Some("alice".into()),
..Default::default()
});
assert!(
resource
.is_match_with_resolver("photos/input/alice.png", &conditions, Some(&resolver))
.await,
"C07_LITERAL_SCOPE_BOUND: actual policy engine substitutes resource variables"
);
assert!(
!literal("input/${aws:username}.png"),
"C07_LITERAL_SCOPE_BOUND: requested resources must stay literal"
);
let restriction = rustfs_iam::delegation::Restrictions {
deployment: "runner".into(),
bucket: "photos".into(),
input_prefix: "input/${aws:username}/".into(),
output_prefix: "output/".into(),
expires_at_unix_seconds: 200,
};
assert!(
restriction.validate(100).is_err(),
"C07_LITERAL_SCOPE_BOUND: legacy stored variable restrictions are rejected"
);
}
}
@@ -0,0 +1,167 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Restriction registration only. These endpoints do not vend credentials.
use crate::admin::{
auth::{AdminResourceScope, validate_admin_action_with_bucket_object_for_iam},
router::{AdminOperation, Operation, S3Router},
runtime_sources,
};
use crate::{
auth::{check_key_valid_with_context, get_session_token},
server::{ADMIN_PREFIX, RemoteAddr},
};
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use matchit::Params;
use rustfs_iam::delegation::{DelegationError, Restrictions};
use rustfs_policy::policy::action::{Action, AdminAction};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
pub fn register_plugin_delegation_route(router: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
router.insert(Method::POST, &format!("{ADMIN_PREFIX}/v3/plugin-delegations"), AdminOperation(&Create))?;
router.insert(Method::GET, &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}"), AdminOperation(&Get))?;
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/revoke"),
AdminOperation(&Revoke),
)?;
Ok(())
}
pub(super) fn storage_error(error: DelegationError) -> s3s::S3Error {
match error {
DelegationError::Invalid => s3_error!(InvalidRequest, "invalid delegation restriction"),
DelegationError::NotFound | DelegationError::OwnerRejected => s3_error!(NoSuchKey, "delegation not found"),
DelegationError::Unavailable => s3_error!(InternalError, "delegation unavailable"),
_ => s3_error!(AccessDenied, "delegation rejected"),
}
}
pub(super) async fn owner(
req: &S3Request<Body>,
action: AdminAction,
) -> S3Result<(std::sync::Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>, String)> {
let signed = req
.credentials
.as_ref()
.ok_or_else(|| s3_error!(AccessDenied, "authentication required"))?;
if get_session_token(&req.uri, &req.headers).is_some() {
return Err(s3_error!(AccessDenied, "original user required"));
}
let context =
runtime_sources::app_context_from_req(req).ok_or_else(|| s3_error!(InternalError, "application context unavailable"))?;
let (credentials, root) = check_key_valid_with_context("", &signed.access_key, Some(context.as_ref())).await?;
if root || credentials.is_temp() || credentials.is_service_account() {
return Err(s3_error!(AccessDenied, "original user required"));
}
if !context.iam().is_ready() {
return Err(s3_error!(InternalError, "iam unavailable"));
}
let iam = context.iam().handle();
validate_admin_action_with_bucket_object_for_iam(
iam.clone(),
&req.headers,
&credentials,
false,
Action::AdminAction(action),
req.extensions.get::<Option<RemoteAddr>>().and_then(|a| a.map(|a| a.0)),
AdminResourceScope::bucket_object("", ""),
)
.await?;
Ok((iam, credentials.access_key))
}
pub(super) fn response<T: serde::Serialize>(value: &T) -> S3Result<S3Response<(StatusCode, Body)>> {
let bytes = serde_json::to_vec(value).map_err(|_| s3_error!(InternalError, "delegation response unavailable"))?;
let mut headers = HeaderMap::new();
headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json"));
headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
Ok(S3Response::with_headers((StatusCode::OK, Body::from(bytes)), headers))
}
struct Create;
#[async_trait::async_trait]
impl Operation for Create {
async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, owner) = owner(&req, AdminAction::CreatePluginDelegationAction).await?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3_error!(InvalidRequest, "delegation body unavailable"))?
.map_err(|_| s3_error!(InvalidRequest, "delegation body unavailable"))?;
let restriction: Restrictions =
serde_json::from_slice(&bytes).map_err(|_| s3_error!(InvalidRequest, "invalid delegation restriction"))?;
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|_| s3_error!(InternalError, "clock unavailable"))?
.as_secs();
let record = iam
.create_plugin_delegation(&owner, restriction, now)
.await
.map_err(storage_error)?;
response(&record)
}
}
struct Get;
#[async_trait::async_trait]
impl Operation for Get {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, owner) = owner(&req, AdminAction::GetPluginDelegationAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3_error!(InvalidRequest, "delegation id required"))?;
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|_| s3_error!(InternalError, "clock unavailable"))?
.as_secs();
let record = iam
.read_active_plugin_delegation(&owner, id, now)
.await
.map_err(storage_error)?;
response(&record)
}
}
struct Revoke;
#[async_trait::async_trait]
impl Operation for Revoke {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, owner) = owner(&req, AdminAction::RevokePluginDelegationAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3_error!(InvalidRequest, "delegation id required"))?;
iam.revoke_plugin_delegation(&owner, id).await.map_err(storage_error)?;
response(&serde_json::json!({"revoked":true,"id":id}))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn delegation_mutations_require_authenticated_original_user() {
for operation in 0..3 {
let req = S3Request {
input: Body::from("{}".to_owned()),
method: Method::POST,
uri: http::Uri::from_static("/rustfs/admin/v3/plugin-delegations"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let result = if operation == 0 {
Get.call(req, Params::new()).await
} else if operation == 1 {
Revoke.call(req, Params::new()).await
} else {
Create.call(req, Params::new()).await
};
let error = result.expect_err("unsigned mutations must not reach persistence");
assert_eq!(error.code(), &s3s::S3ErrorCode::AccessDenied);
}
}
}
+2
View File
@@ -64,6 +64,8 @@ pub fn make_admin_route(
fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
health::register_health_route(r)?;
sts::register_admin_auth_route(r)?;
handlers::plugin_delegation::register_plugin_delegation_route(r)?;
handlers::plugin_attempt::register_plugin_attempt_route(r)?;
account::register_account_route(r)?;
mfa::register_mfa_route(r)?;
+28
View File
@@ -23,6 +23,10 @@ const ATTACH_POLICY: AdminActionRef = AdminActionRef::new("AttachPolicyAdminActi
const CONFIG_UPDATE: AdminActionRef = AdminActionRef::new("ConfigUpdateAdminAction");
const CONSOLE_LOG: AdminActionRef = AdminActionRef::new("ConsoleLogAdminAction");
const COMMIT_TABLE: AdminActionRef = AdminActionRef::new("CommitTableAction");
const ISSUE_PLUGIN_ATTEMPT: AdminActionRef = AdminActionRef::new("IssuePluginAttemptAction");
const GET_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("GetPluginDelegationAction");
const CREATE_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("CreatePluginDelegationAction");
const REVOKE_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("RevokePluginDelegationAction");
const CREATE_POLICY: AdminActionRef = AdminActionRef::new("CreatePolicyAdminAction");
const CREATE_USER: AdminActionRef = AdminActionRef::new("CreateUserAdminAction");
const CREATE_SERVICE_ACCOUNT: AdminActionRef = AdminActionRef::new("CreateServiceAccountAdminAction");
@@ -636,6 +640,30 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
SERVER_INFO,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/issue-attempt",
ISSUE_PLUGIN_ATTEMPT,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v3/plugin-delegations/{id}",
GET_PLUGIN_DELEGATION,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations",
CREATE_PLUGIN_DELEGATION,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/revoke",
REVOKE_PLUGIN_DELEGATION,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v4/plugins/catalog",
+32
View File
@@ -33,6 +33,7 @@ use std::sync::{Arc, OnceLock};
#[derive(Default)]
pub struct ServerContextSlot {
app_context: OnceLock<Arc<AppContext>>,
capture_service: std::sync::Mutex<Option<Arc<crate::startup_capture::NativeCaptureService>>>,
bootstrap_target: Option<BootstrapLocalTarget>,
heal_topology_fingerprint: Arc<tokio::sync::OnceCell<String>>,
}
@@ -51,6 +52,7 @@ impl ServerContextSlot {
pub fn new() -> Arc<Self> {
Arc::new(Self {
app_context: OnceLock::new(),
capture_service: std::sync::Mutex::new(None),
bootstrap_target: None,
heal_topology_fingerprint: Arc::new(tokio::sync::OnceCell::new()),
})
@@ -94,6 +96,36 @@ impl ServerContextSlot {
self.bootstrap_target.clone()
}
pub(crate) fn install_capture_service(
&self,
service: Arc<crate::startup_capture::NativeCaptureService>,
) -> std::io::Result<()> {
let mut installed = self
.capture_service
.lock()
.map_err(|_| std::io::Error::other("capture service slot poisoned"))?;
if installed.is_some() {
return Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
"capture service already installed",
));
}
*installed = Some(service);
Ok(())
}
pub(crate) fn installed_capture_service(&self) -> Option<Arc<crate::startup_capture::NativeCaptureService>> {
self.capture_service.lock().ok()?.clone()
}
pub(crate) fn remove_capture_service(&self, service: &Arc<crate::startup_capture::NativeCaptureService>) {
if let Ok(mut installed) = self.capture_service.lock()
&& installed.as_ref().is_some_and(|current| Arc::ptr_eq(current, service))
{
installed.take();
}
}
/// This server's installed application context, if startup has completed.
///
/// Request-scoped resolution must use this accessor so an empty slot never
+5
View File
@@ -1593,6 +1593,10 @@ pub struct ServerOpts {
#[arg(long, env = "RUSTFS_TLS_PATH")]
pub tls_path: Option<String>,
/// Configuration file for the optional native capture Raft runtime.
#[arg(long, env = "RUSTFS_NATIVE_CAPTURE_CONFIG")]
pub native_capture_config: Option<PathBuf>,
#[arg(long, env = "RUSTFS_LICENSE")]
pub license: Option<String>,
@@ -1711,6 +1715,7 @@ pub fn default_server_opts() -> ServerOpts {
console_address: DEFAULT_CONSOLE_ADDRESS.to_string(),
obs_endpoint: DEFAULT_OBS_ENDPOINT.to_string(),
tls_path: None,
native_capture_config: None,
license: None,
region: None,
kms_enable: false,
+7
View File
@@ -24,6 +24,7 @@ use rustfs_config::{
};
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, Masked};
use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::{Mutex, OnceLock};
pub(crate) const LEGACY_ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
@@ -110,6 +111,9 @@ pub struct Config {
/// tls path for rustfs API and console.
pub tls_path: Option<String>,
/// Optional native capture Raft configuration file.
pub native_capture_config: Option<PathBuf>,
/// License key for enterprise features
pub license: Option<String>,
@@ -166,6 +170,7 @@ impl Config {
console_address: DEFAULT_CONSOLE_ADDRESS.to_string(),
obs_endpoint: rustfs_config::DEFAULT_OBS_ENDPOINT.to_string(),
tls_path: None,
native_capture_config: None,
license: None,
region: Some(RUSTFS_REGION.to_string()),
kms_enable: false,
@@ -200,6 +205,7 @@ impl Config {
console_address,
obs_endpoint,
tls_path,
native_capture_config,
license,
region,
kms_enable,
@@ -243,6 +249,7 @@ impl Config {
console_address,
obs_endpoint,
tls_path,
native_capture_config,
license,
region,
kms_enable,
+29
View File
@@ -289,6 +289,7 @@ mod tests {
assert_eq!(config.console_address, DEFAULT_CONSOLE_ADDRESS);
assert_eq!(config.obs_endpoint, DEFAULT_OBS_ENDPOINT);
assert_eq!(config.tls_path, None);
assert_eq!(config.native_capture_config, None);
assert_eq!(config.license, None);
assert_eq!(config.region, Some(RUSTFS_REGION.to_string()));
assert!(!config.kms_enable);
@@ -304,6 +305,34 @@ mod tests {
assert_eq!(config.buffer_profile, "GeneralPurpose");
}
#[test]
#[serial]
fn test_native_capture_config_entry() {
with_env_var("RUSTFS_NATIVE_CAPTURE_CONFIG", "/tmp/native-capture-env.json", || {
let opt = Opt::parse_from(["rustfs", "server", "/test/volume"]);
assert_eq!(opt.native_capture_config, Some("/tmp/native-capture-env.json".into()));
let config = Config::from_opt(opt).expect("environment capture configuration path");
assert_eq!(config.native_capture_config, Some("/tmp/native-capture-env.json".into()));
let opt = Opt::parse_from([
"rustfs",
"/test/volume",
"--native-capture-config",
"/tmp/native-capture-cli.json",
]);
let config = Config::from_opt(opt).expect("CLI capture configuration path");
assert_eq!(config.native_capture_config, Some("/tmp/native-capture-cli.json".into()));
});
}
#[test]
#[serial]
fn test_native_capture_config_default_disabled() {
let opt = Opt::from_server_opts(default_server_opts());
assert_eq!(opt.native_capture_config, None);
assert_eq!(Config::from_opt(opt).expect("default configuration").native_capture_config, None);
}
#[test]
#[serial]
fn test_custom_console_configuration() {
+2
View File
@@ -42,6 +42,7 @@ pub struct Opt {
pub console_address: String,
pub obs_endpoint: String,
pub tls_path: Option<String>,
pub native_capture_config: Option<PathBuf>,
pub license: Option<String>,
pub region: Option<String>,
pub kms_enable: bool,
@@ -72,6 +73,7 @@ impl Opt {
console_address: o.console_address,
obs_endpoint: o.obs_endpoint,
tls_path: o.tls_path,
native_capture_config: o.native_capture_config,
license: o.license,
region: o.region,
kms_enable: o.kms_enable,
+1
View File
@@ -107,6 +107,7 @@ pub(crate) mod startup_audit;
pub(crate) mod startup_auth;
pub(crate) mod startup_background;
pub(crate) mod startup_bucket_metadata;
pub(crate) mod startup_capture;
pub(crate) mod startup_deadlock;
pub(crate) mod startup_embedded;
pub(crate) mod startup_embedded_optional;
+294 -7
View File
@@ -2277,13 +2277,15 @@ fn process_connection(
.max_encoding_message_size(tier_mutation_control_max_message_size),
check_auth,
);
let rpc_service = RpcRequestPathService::new(
Routes::new(node_service)
.add_service(InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth))
.add_service(heal_control_service)
.add_service(tier_mutation_control_service)
.prepare(),
);
let mut rpc_routes = Routes::new(node_service)
.add_service(InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth))
.add_service(heal_control_service)
.add_service(tier_mutation_control_service);
// Capture verifies its own replay nonce; check_auth must not consume it twice.
if let Some(capture) = server_ctx.installed_capture_service() {
rpc_routes = rpc_routes.add_service(capture.to_service());
}
let rpc_service = RpcRequestPathService::new(rpc_routes.prepare());
#[cfg(feature = "swift")]
let http_service = SwiftService::new(true, None, s3_service);
@@ -4157,6 +4159,291 @@ mod tests {
);
}
#[tokio::test]
#[serial_test::serial]
async fn native_stream_prepare_remote_restart() {
use crate::storage::rpc::node_service::make_server_for_context;
use crate::storage::storage_api::ecstore_disk::DiskAPI as _;
use crate::storage::storage_api::{
ECStore, InstanceContext, bootstrap_instance_ctx, ecstore_storage::init_local_disks_with_instance_ctx,
};
use rustfs_ecstore::api::{
error::is_err_object_not_found,
object::{ObjectOptions, PutObjReader},
};
use rustfs_iam::store::Store as _;
use rustfs_storage_api::object::{ListOperations, ObjectOperations};
use sha2::Digest as _;
use tokio_util::sync::CancellationToken;
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("bind native preparation peer: {err}"),
};
let addr = listener.local_addr().expect("peer listener address");
let previous_node_name = rustfs_common::get_global_local_node_name().await;
rustfs_common::set_global_local_node_name(&addr.to_string()).await;
let _ = rustfs_credentials::set_global_rpc_secret("native-stream-remote-restart-secret".to_string());
let root = tempfile::tempdir().expect("native preparation root");
let env = rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(root.path())
.init_bucket_metadata(true)
.build()
.await;
let bucket = "native-remote-restart";
let key = "large/private-object.bin";
env.make_bucket(bucket, false).await;
let mut local_pool = env.endpoint_pools.as_ref()[0].clone();
let local_endpoints = env
.disk_paths
.iter()
.enumerate()
.map(|(index, path)| {
let mut endpoint =
Endpoint::try_from(format!("http://{addr}{}", path.display()).as_str()).expect("local peer endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(index);
endpoint.is_local = true;
endpoint
})
.collect::<Vec<_>>();
local_pool.endpoints = Endpoints::from(local_endpoints);
let owner_pools = EndpointServerPools::from(vec![local_pool]);
let owner_instance = bootstrap_instance_ctx();
init_local_disks_with_instance_ctx(&owner_instance, owner_pools.clone())
.await
.expect("register the production local disk owners");
let object_store = rustfs_iam::store::object::ObjectStore::new(env.ecstore.clone());
object_store
.save_iam_config(
serde_json::json!({"version": 1}),
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
)
.await
.expect("seed IAM metadata for the node service");
let iam = rustfs_iam::build_iam_sys(env.ecstore.clone())
.await
.expect("build fixture IAM");
let app_context = Arc::new(crate::runtime_sources::AppContext::with_default_interfaces(
env.ecstore.clone(),
iam,
Arc::new(rustfs_kms::KmsServiceManager::new()),
));
let node_service = make_server_for_context(Some(app_context));
for endpoint in owner_pools.as_ref()[0].endpoints.as_ref() {
let disk = crate::storage::storage_api::find_local_disk_by_ref(&endpoint.to_string())
.await
.expect("resolve the same registered disk used by production HTTP RPC");
if let Err(error) = disk
.make_volume(crate::storage::storage_api::ecstore_disk::RUSTFS_META_BUCKET)
.await
{
assert_eq!(
error,
crate::storage::storage_api::DiskError::VolumeExists,
"create owner metadata volume"
);
}
}
let node = InterceptedService::new(
NodeServiceServer::new(node_service)
.max_decoding_message_size(rustfs_protos::internode_rpc_max_message_size())
.max_encoding_message_size(rustfs_protos::internode_rpc_max_message_size()),
check_auth,
);
let grpc = RpcRequestPathService::new(Routes::new(node).prepare());
let http =
InternodeRpcService::new(tower::service_fn(|_| async { Ok::<_, Infallible>(Response::new(s3s::Body::empty())) }));
let service = hybrid(http, grpc);
let shutdown = tokio_util::sync::CancellationToken::new();
let shutdown_for_task = shutdown.clone();
let server = tokio::spawn(async move {
loop {
tokio::select! {
_ = shutdown_for_task.cancelled() => break,
accepted = listener.accept() => {
let (socket, _) = accepted.expect("accept native preparation peer connection");
let service = service.clone();
tokio::spawn(async move {
let _ = ConnBuilder::new(TokioExecutor::new())
.serve_connection(TokioIo::new(socket), TowerToHyperService::new(service))
.await;
});
}
}
}
});
let mut remote_pool = env.endpoint_pools.as_ref()[0].clone();
let remote_endpoints = env
.disk_paths
.iter()
.enumerate()
.map(|(index, path)| {
let mut endpoint =
Endpoint::try_from(format!("http://{addr}{}", path.display()).as_str()).expect("remote peer endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(index);
endpoint.is_local = false;
endpoint
})
.collect::<Vec<_>>();
remote_pool.endpoints = Endpoints::from(remote_endpoints);
let remote_pools = EndpointServerPools::from(vec![remote_pool]);
let client_instance = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&client_instance, remote_pools.clone())
.await
.expect("connect production RemoteDisk clients");
let client_cancel = CancellationToken::new();
let client = ECStore::new_with_instance_ctx(addr, remote_pools.clone(), client_cancel.clone(), client_instance)
.await
.expect("open coordinator over real RemoteDisk clients");
assert_eq!(client.id, env.ecstore.id, "remote coordinator must use the same deployment identity");
let preparation = uuid::Uuid::new_v4();
let body = (0..(5 * 1024 * 1024 + 37))
.map(|index| u8::try_from(index % 251).expect("bounded byte"))
.collect::<Vec<_>>();
let expected_digest: [u8; 32] = sha2::Sha256::digest(&body).into();
let prepared = client
.prepare_native_object(bucket, key, preparation, &mut PutObjReader::from_vec(body.clone()))
.await
.expect("authenticated TCP streaming preparation over remote disks");
assert_eq!(prepared.object_digest(), expected_digest);
assert!(prepared.receipt_count() >= 3, "remote write quorum must be distinct and durable");
assert_eq!(
prepared.disk_ids().collect::<std::collections::HashSet<_>>().len(),
prepared.receipt_count(),
"remote quorum must name distinct physical disks"
);
let expected_receipt_count = prepared.receipt_count();
let expected_disk_ids = prepared.disk_ids().collect::<std::collections::HashSet<_>>();
let mut durable_disk_count = 0;
for path in &env.disk_paths {
let record = path.join(".rustfs.sys/native-preparations").join(preparation.to_string());
if record.exists() {
assert!(record.join("PREPARED").is_file(), "owner disk persisted the prepared marker");
assert!(record.join(preparation.to_string()).join("part.1").is_file());
durable_disk_count += 1;
}
}
assert!(durable_disk_count >= 3, "real remote quorum must exist on owner disks");
drop(prepared);
assert!(
client
.get_object_info(bucket, key, &ObjectOptions::default())
.await
.is_err_and(|error| is_err_object_not_found(&error))
);
let listed = client
.clone()
.list_objects_v2(bucket, "large/", None, None, 100, false, None, false)
.await
.expect("ordinary LIST remains available");
assert!(
listed.objects.is_empty() && listed.prefixes.is_empty(),
"private preparation must stay out of ordinary LIST"
);
client_cancel.cancel();
drop(client);
tokio::task::yield_now().await;
let restarted_instance = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&restarted_instance, remote_pools.clone())
.await
.expect("reconnect RemoteDisk clients after coordinator restart");
let restarted_cancel = CancellationToken::new();
let restarted = ECStore::new_with_instance_ctx(addr, remote_pools, restarted_cancel.clone(), restarted_instance)
.await
.expect("restart coordinator over durable remote disks");
let recovered = restarted
.recover_native_object(bucket, key, preparation)
.await
.expect("recover exact remote preparation after restart");
assert_eq!(recovered.object_digest(), expected_digest);
assert_eq!(recovered.object_length(), u64::try_from(body.len()).expect("bounded length"));
assert_eq!(recovered.receipt_count(), expected_receipt_count);
assert_eq!(recovered.disk_ids().collect::<std::collections::HashSet<_>>(), expected_disk_ids);
assert!(
restarted
.get_object_info(bucket, key, &ObjectOptions::default())
.await
.is_err_and(|error| is_err_object_not_found(&error))
);
let listed = restarted
.clone()
.list_objects_v2(bucket, "large/", None, None, 100, false, None, false)
.await
.expect("ordinary LIST remains available after restart");
assert!(
listed.objects.is_empty() && listed.prefixes.is_empty(),
"recovered preparation remains private"
);
drop(recovered);
assert_eq!(durable_disk_count, 4, "failure cases require all four owner disks to be prepared");
let record_dir = env.disk_paths[0]
.join(".rustfs.sys/native-preparations")
.join(preparation.to_string());
let intent_path = record_dir.join("intent.json");
let marker_path = record_dir.join("PREPARED");
let original_intent = std::fs::read(&intent_path).expect("read durable remote intent");
let original_marker = std::fs::read(&marker_path).expect("read durable remote marker");
let mut invalid_marker = original_marker.clone();
invalid_marker[0] ^= 1;
let mut intent: serde_json::Value = serde_json::from_slice(&original_intent).expect("decode remote intent");
let original_set_digest = serde_json::to_string(&intent["set_digest"]).expect("serialize retained set digest");
let byte = intent["set_digest"][0].as_u64().expect("set digest byte");
intent["set_digest"][0] = serde_json::json!(byte ^ 1);
let changed_set_digest = serde_json::to_string(&intent["set_digest"]).expect("serialize conflicting set digest");
// Preserve the typed record's canonical field order while changing its physical identity.
let conflicting_intent = String::from_utf8(original_intent.clone())
.expect("UTF-8 intent")
.replacen(
&format!("\"set_digest\":{original_set_digest}"),
&format!("\"set_digest\":{changed_set_digest}"),
1,
)
.into_bytes();
assert_ne!(conflicting_intent, original_intent, "inject a real physical set identity conflict");
let conflicting_marker = sha2::Sha256::digest(&conflicting_intent).to_vec();
for (intent, marker) in [
(&original_intent, &invalid_marker),
(&conflicting_intent, &conflicting_marker),
] {
std::fs::write(&intent_path, intent).expect("inject remote intent state");
std::fs::write(&marker_path, marker).expect("inject remote marker state");
let result = restarted.recover_native_object(bucket, key, preparation).await;
std::fs::write(&intent_path, &original_intent).expect("restore remote intent");
std::fs::write(&marker_path, &original_marker).expect("restore remote marker");
assert!(
result.is_err_and(|error| matches!(error, rustfs_ecstore::api::error::StorageError::FileCorrupt)),
"one corrupt remote owner must fail recovery even when three healthy disks form a quorum"
);
}
let unavailable_path = root.path().join("unavailable-native-disk");
std::fs::rename(&env.disk_paths[0], &unavailable_path).expect("take one owner disk offline");
let degraded = restarted.recover_native_object(bucket, key, preparation).await;
std::fs::rename(&unavailable_path, &env.disk_paths[0]).expect("restore offline owner disk");
let degraded = degraded.expect("three healthy remote owners must retain the write quorum");
assert_eq!(degraded.receipt_count(), 3);
assert_eq!(degraded.object_digest(), expected_digest);
assert_eq!(degraded.object_length(), u64::try_from(body.len()).expect("bounded length"));
drop(degraded);
restarted_cancel.cancel();
shutdown.cancel();
let _ = server.await;
rustfs_common::set_global_local_node_name(&previous_node_name).await;
}
#[tokio::test]
#[serial_test::serial]
async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() {
+295
View File
@@ -0,0 +1,295 @@
//! Opt-in native capture ownership within the RustFS listener lifecycle.
mod config;
#[cfg(all(test, any(target_os = "linux", target_os = "macos")))]
mod tests;
use crate::{config::Config, storage_api::server::http::ServerContextSlot};
use config::CaptureConfig;
#[cfg(any(target_os = "linux", target_os = "macos"))]
use config::CaptureMode;
#[cfg(any(target_os = "linux", target_os = "macos"))]
use rustfs_capture_authority::Store;
use rustfs_capture_authority::{CheckpointLimits, NativeLogWriter, NativeRaftOwner};
use rustfs_ecstore::api::rpc::{CaptureRaftLimits, CaptureRaftNetworkFactory, CaptureRaftRpc, normalize_tonic_rpc_audience};
use rustfs_protos::proto_gen::node_service::{
CaptureRaftRequest, CaptureRaftResponse,
capture_raft_service_server::{CaptureRaftService, CaptureRaftServiceServer},
};
use std::{io, sync::Arc};
use tonic::{Request, Response, Status};
const EVENT_NATIVE_CAPTURE_STATE: &str = "native_capture_state";
const LOG_COMPONENT_MAIN: &str = "main";
const LOG_SUBSYSTEM_STARTUP: &str = "startup";
/// Idle HTTP routes retain this holder, never the native storage owner after unmount.
#[derive(Clone)]
pub(crate) struct NativeCaptureService {
rpc: Arc<tokio::sync::RwLock<Option<CaptureRaftRpc>>>,
limits: CaptureRaftLimits,
accepting: Arc<std::sync::atomic::AtomicBool>,
}
impl NativeCaptureService {
pub(crate) fn to_service(&self) -> CaptureRaftServiceServer<Self> {
CaptureRaftServiceServer::new(self.clone())
.max_decoding_message_size(self.limits.max_request_bytes + 64)
.max_encoding_message_size(self.limits.max_response_bytes + 128)
}
}
#[tonic::async_trait]
impl CaptureRaftService for NativeCaptureService {
async fn append_entries(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
return Err(Status::unavailable("native capture service stopping"));
}
let rpc = self.rpc.read().await;
rpc.as_ref()
.ok_or_else(|| Status::unavailable("native capture service stopped"))?
.append_entries(request)
.await
}
async fn vote(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
return Err(Status::unavailable("native capture service stopping"));
}
let rpc = self.rpc.read().await;
rpc.as_ref()
.ok_or_else(|| Status::unavailable("native capture service stopped"))?
.vote(request)
.await
}
async fn install_snapshot(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
return Err(Status::unavailable("native capture service stopping"));
}
let rpc = self.rpc.read().await;
rpc.as_ref()
.ok_or_else(|| Status::unavailable("native capture service stopped"))?
.install_snapshot(request)
.await
}
}
pub(crate) struct NativeCaptureRuntime {
owner: Option<Arc<NativeRaftOwner>>,
writer: Option<NativeLogWriter>,
raft_config: Arc<openraft::Config>,
factory: CaptureRaftNetworkFactory,
service: Arc<NativeCaptureService>,
slot: Arc<ServerContextSlot>,
config: CaptureConfig,
#[cfg(any(target_os = "linux", target_os = "macos"))]
reader_shutdown: rustfs_capture_authority::source_reader::Shutdown,
#[cfg(any(target_os = "linux", target_os = "macos"))]
reader_task: tokio::task::JoinHandle<Result<(), rustfs_capture_authority::source_reader::ReaderError>>,
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
pub(crate) async fn start_capture(config: &Config, slot: &Arc<ServerContextSlot>) -> io::Result<Option<NativeCaptureRuntime>> {
let Some(path) = &config.native_capture_config else {
return Ok(None);
};
let path = path.clone();
let capture = tokio::task::spawn_blocking(move || CaptureConfig::read(path))
.await
.map_err(io::Error::other)??;
let address = rustfs_utils::net::parse_and_resolve_address(&config.address).map_err(io::Error::other)?;
if address.port() == 0 {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"native capture requires a fixed listener port",
));
}
// Resolve the existing internode credential before creating durable state.
rustfs_credentials::try_get_rpc_token()?;
let raft_config = Arc::new(openraft::Config::default().validate().map_err(io::Error::other)?);
let limits = CheckpointLimits::default();
let transport_limits = CaptureRaftLimits::default();
let factory = CaptureRaftNetworkFactory::new(capture.identity, capture.peers.clone(), raft_config.clone(), transport_limits)
.map_err(io::Error::other)?;
let database = capture.database.clone();
let identity = capture.identity;
let binding = capture.capture_binding.clone();
let mode = capture.mode;
let store = tokio::task::spawn_blocking(move || {
let mut store = match mode {
CaptureMode::Create => Store::initialize(database, identity)?,
CaptureMode::Recover => Store::open_with_checkpoint_limits(database, identity, limits)?,
};
match mode {
CaptureMode::Create => {
store.pin_capture_binding(binding)?;
store.enable_native_empty()?;
store.promote_created_provenance()?;
}
CaptureMode::Recover => {
if store.capture_binding()? != Some(binding) {
return Err(rustfs_capture_authority::StoreError::BindingConflict);
}
store.validate_native_provenance_state()?;
}
}
Ok::<_, rustfs_capture_authority::StoreError>(store)
})
.await
.map_err(io::Error::other)?
.map_err(io::Error::other)?;
let writer = NativeLogWriter::new(store);
// Refuse an occupied reader path before Raft can perform any background work.
let (reader, reader_shutdown) =
rustfs_capture_authority::source_reader::SourceReaderServer::bind(&capture.reader_socket, writer.clone())
.map_err(io::Error::other)?;
let service = Arc::new(NativeCaptureService {
rpc: Arc::new(tokio::sync::RwLock::new(None)),
limits: transport_limits,
accepting: Arc::new(std::sync::atomic::AtomicBool::new(true)),
});
let runtime = NativeCaptureRuntime {
owner: None,
writer: Some(writer),
raft_config,
factory,
service,
slot: slot.clone(),
config: capture,
reader_shutdown,
reader_task: tokio::spawn(reader.serve()),
};
if let Err(error) = slot.install_capture_service(runtime.service.clone()) {
runtime.shutdown().await?;
return Err(error);
}
Ok(Some(runtime))
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
pub(crate) async fn start_capture(config: &Config, _slot: &Arc<ServerContextSlot>) -> io::Result<Option<NativeCaptureRuntime>> {
if config.native_capture_config.is_some() {
return Err(io::Error::new(io::ErrorKind::Unsupported, "native capture requires Linux or macOS"));
}
Ok(None)
}
impl NativeCaptureRuntime {
#[cfg(test)]
fn owner(&self) -> &NativeRaftOwner {
self.owner.as_deref().expect("startup finished with a real native owner")
}
pub(crate) fn stop_admission(&self) {
self.service.accepting.store(false, std::sync::atomic::Ordering::Release);
}
pub(crate) async fn finish_startup(&mut self, audience: &str) -> io::Result<()> {
let local = self
.config
.peers
.get(&self.config.identity.node_id)
.ok_or_else(|| io::Error::other("native capture local member missing"))?;
let uri: http::Uri = local.rpc_uri().parse().map_err(io::Error::other)?;
let configured = normalize_tonic_rpc_audience(
uri.authority()
.ok_or_else(|| io::Error::other("native capture local authority missing"))?
.as_str(),
)?;
if configured != normalize_tonic_rpc_audience(audience)? {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"native capture local authority differs from Ping audience",
));
}
if !self.service.accepting.load(std::sync::atomic::Ordering::Acquire) {
return Err(io::Error::other("native capture service stopping"));
}
let writer = self
.writer
.take()
.ok_or_else(|| io::Error::other("native capture owner already started"))?;
// Raft::new can replay committed entries and restore leader replication.
// Neither may run until this process's actual Ping audience is verified.
let owner = Arc::new(
NativeRaftOwner::start(
writer,
self.config.identity.node_id,
self.raft_config.clone(),
CheckpointLimits::default(),
self.factory.clone(),
)
.await
.map_err(io::Error::other)?,
);
self.owner = Some(owner.clone());
let rpc = CaptureRaftRpc::new(owner.clone(), self.config.peers.clone(), CaptureRaftLimits::default())
.map_err(io::Error::other)?;
*self.service.rpc.write().await = Some(rpc);
if self.config.bootstrap {
owner.bootstrap(self.config.peers.clone()).await.map_err(io::Error::other)?;
}
tracing::info!(
event = EVENT_NATIVE_CAPTURE_STATE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "ready",
node_id = self.config.identity.node_id,
mode = ?self.config.mode,
"Native capture runtime ready"
);
Ok(())
}
pub(crate) async fn shutdown(self) -> io::Result<()> {
let Self {
owner,
writer,
raft_config: _,
factory: _,
service,
slot,
config: _,
#[cfg(any(target_os = "linux", target_os = "macos"))]
reader_shutdown,
#[cfg(any(target_os = "linux", target_os = "macos"))]
reader_task,
} = self;
service.accepting.store(false, std::sync::atomic::Ordering::Release);
slot.remove_capture_service(&service);
let rpc = service.rpc.write().await.take();
let rpc_result = match &rpc {
Some(rpc) => rpc.shutdown().await.map_err(io::Error::other),
None => Ok(()),
};
let reader_result = {
#[cfg(any(target_os = "linux", target_os = "macos"))]
{
reader_shutdown.shutdown();
reader_task
.await
.map_err(io::Error::other)
.and_then(|result| result.map_err(io::Error::other))
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
{
Ok::<_, io::Error>(())
}
};
let owner_result = match &owner {
Some(owner) => owner.shutdown().await.map_err(io::Error::other),
None => Ok(()),
};
drop(rpc);
drop(service);
drop(owner);
drop(writer);
rpc_result?;
reader_result?;
owner_result?;
tracing::info!(
event = EVENT_NATIVE_CAPTURE_STATE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopped",
"Native capture runtime drained"
);
Ok(())
}
}
+150
View File
@@ -0,0 +1,150 @@
//! Bounded configuration for the opt-in native capture runtime.
use rustfs_capture_authority::{CaptureBinding, CaptureNode, NodeIdentity};
use serde::Deserialize;
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{self, Read};
use std::path::{Path, PathBuf};
const MAX_CONFIG_BYTES: u64 = 64 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum CaptureMode {
Create,
Recover,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct CaptureConfig {
pub identity: NodeIdentity,
pub database: PathBuf,
pub reader_socket: PathBuf,
pub peers: BTreeMap<u64, CaptureNode>,
pub capture_binding: CaptureBinding,
pub mode: CaptureMode,
pub bootstrap: bool,
}
impl CaptureConfig {
pub(super) fn read(path: impl AsRef<Path>) -> io::Result<Self> {
let file = File::open(path)?;
let mut bytes = Vec::new();
file.take(MAX_CONFIG_BYTES + 1).read_to_end(&mut bytes)?;
if bytes.len() as u64 > MAX_CONFIG_BYTES {
return Err(invalid("native capture configuration exceeds 64 KiB"));
}
let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid("invalid native capture configuration JSON"))?;
config.validate()?;
Ok(config)
}
fn validate(&self) -> io::Result<()> {
if self.identity.cluster_id == [0; 16]
|| self.capture_binding.binding_id == [0; 16]
|| self.capture_binding.target_id == [0; 16]
{
return Err(invalid("native capture identifiers must not be nil"));
}
if self.database.as_os_str().is_empty() || self.reader_socket.as_os_str().is_empty() {
return Err(invalid("native capture paths must not be empty"));
}
if self.database == self.reader_socket {
return Err(invalid("native capture database and reader socket paths must differ"));
}
if !self.peers.contains_key(&self.identity.node_id) {
return Err(invalid("native capture peers must include the local node"));
}
for peer in self.peers.values() {
peer.validate().map_err(|_| invalid("invalid native capture peer"))?;
}
if self.bootstrap && self.mode != CaptureMode::Create {
return Err(invalid("native capture bootstrap requires create mode"));
}
Ok(())
}
}
fn invalid(message: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, message)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::{Value, json};
use std::io::Write;
fn configuration() -> Value {
json!({
"identity": {"cluster_id": ([1u8; 16]), "node_id": 0},
"database": "/tmp/capture.redb",
"reader_socket": "/tmp/capture.sock",
"peers": {"0": {"rpc_uri": "http://127.0.0.1:9000"}},
"capture_binding": {"revision": 1, "binding_id": ([2u8; 16]), "target_id": ([3u8; 16])},
"mode": "create",
"bootstrap": true
})
}
fn read(value: &Value) -> io::Result<CaptureConfig> {
let mut file = tempfile::NamedTempFile::new().expect("configuration file");
file.write_all(&serde_json::to_vec(value).expect("JSON"))
.expect("write configuration");
CaptureConfig::read(file.path())
}
#[test]
fn native_capture_config_reads_create_and_recover() {
let mut value = configuration();
let config = read(&value).expect("create configuration");
assert_eq!(config.mode, CaptureMode::Create);
assert_eq!(config.identity.node_id, 0);
assert!(config.bootstrap);
value["mode"] = json!("recover");
value["bootstrap"] = json!(false);
assert_eq!(read(&value).expect("recover configuration").mode, CaptureMode::Recover);
}
#[test]
fn native_capture_config_rejects_invalid_fields() {
for (field, replacement) in [
("database", json!("")),
("reader_socket", json!("")),
("peers", json!({})),
("mode", json!("recover")),
("unknown", json!(true)),
] {
let mut value = configuration();
value[field] = replacement;
assert_eq!(read(&value).expect_err(field).kind(), io::ErrorKind::InvalidData);
}
for (parent, field) in [
("identity", "cluster_id"),
("capture_binding", "binding_id"),
("capture_binding", "target_id"),
] {
let mut value = configuration();
value[parent][field] = json!(([0u8; 16]));
assert_eq!(read(&value).expect_err(field).kind(), io::ErrorKind::InvalidData);
}
let mut value = configuration();
value["peers"]["0"]["rpc_uri"] = json!("http://127.0.0.1:0");
assert!(read(&value).is_err());
value = configuration();
value["identity"]["unknown"] = json!(true);
assert!(read(&value).is_err());
}
#[test]
fn native_capture_config_bounds_actual_read() {
let mut file = tempfile::NamedTempFile::new().expect("configuration file");
let mut bytes = serde_json::to_vec(&configuration()).expect("JSON");
bytes.resize(MAX_CONFIG_BYTES as usize, b' ');
file.write_all(&bytes).expect("write bounded configuration");
CaptureConfig::read(file.path()).expect("exact limit accepted");
file.write_all(b" ").expect("exceed limit");
assert_eq!(CaptureConfig::read(file.path()).expect_err("oversize").kind(), io::ErrorKind::InvalidData);
}
}
+527
View File
@@ -0,0 +1,527 @@
//! Startup tests exercise the installed native owner and its real source reader.
use super::*;
use rustfs_capture_authority::{
CaptureBinding, CaptureCommand, CaptureNode, DecideCreated, NodeIdentity, ObjectIdentity, PreparedIdentity,
source_reader::SourceReaderClient,
};
use std::collections::BTreeMap;
use std::time::Duration;
#[derive(Debug, PartialEq, Eq)]
struct DurableCaptureState {
identity: NodeIdentity,
binding: Option<CaptureBinding>,
vote: Option<rustfs_capture_authority::VoteRecord>,
committed: Option<rustfs_capture_authority::LogId>,
logs: Vec<rustfs_capture_authority::LogEntry>,
applied_membership: (Option<rustfs_capture_authority::LogId>, rustfs_capture_authority::CaptureMembership),
events_cursor: rustfs_capture_authority::EventPage,
purged: Option<rustfs_capture_authority::LogId>,
tail: Option<rustfs_capture_authority::LogId>,
}
async fn durable_state(database: &std::path::Path, expected: NodeIdentity) -> DurableCaptureState {
use openraft::storage::RaftLogStorage;
let store = Store::open(database, expected).expect("inspect retained store with original identity");
let binding = store.capture_binding().expect("binding");
let vote = store.read_vote().expect("vote");
let committed = store.read_committed().expect("committed");
let logs = store.read_entries(0, u64::MAX).expect("every retained log payload");
let applied_membership = store.native_application_state().expect("applied and membership");
let events_cursor = store
.created_events_page(None, 256)
.expect("complete fixture events and cursor");
let writer = NativeLogWriter::new(store);
let identity = writer.durable_identity().await.expect("actual durable identity");
let mut storage = rustfs_capture_authority::NativeRaftLogStore::new(writer.clone());
let state = storage.get_log_state().await.expect("durable log boundaries");
DurableCaptureState {
identity,
binding,
vote,
committed,
logs,
applied_membership,
events_cursor,
purged: state.last_purged_log_id.map(Into::into),
tail: state.last_log_id.map(Into::into),
}
}
fn capture_config(directory: &std::path::Path, node_id: u64) -> config::CaptureConfig {
config::CaptureConfig {
identity: NodeIdentity {
cluster_id: [1; 16],
node_id,
},
database: directory.join("capture.redb"),
reader_socket: directory.join("capture.sock"),
peers: BTreeMap::from([(
node_id,
CaptureNode::new(format!("http://127.0.0.1:{}", 19000 + node_id)).expect("fixed local peer"),
)]),
capture_binding: CaptureBinding {
revision: 9,
binding_id: [8; 16],
target_id: [9; 16],
},
mode: config::CaptureMode::Create,
bootstrap: true,
}
}
fn command(value: u8) -> CaptureCommand {
CaptureCommand::try_created(DecideCreated {
operation_id: [value; 16],
object: ObjectIdentity {
bucket_incarnation: [2; 16],
key: vec![value],
},
expected_head: None,
prepared: PreparedIdentity {
version_id: [value; 16],
preparation_id: [4; 16],
content_digest: [5; 32],
content_length: 123,
},
binding_revision: 9,
})
.expect("real native command")
}
fn private_directory() -> tempfile::TempDir {
let _ = rustfs_credentials::set_global_rpc_secret("native-capture-startup-test-secret".into());
rustfs_credentials::try_get_rpc_token().expect("configured internal RPC secret");
use std::os::unix::fs::PermissionsExt;
let directory = tempfile::tempdir().expect("startup storage directory");
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).expect("private reader parent");
directory
}
fn server_config(capture: &config::CaptureConfig) -> crate::config::Config {
let path = capture.database.with_extension("json");
let value = serde_json::json!({
"identity": capture.identity,
"database": capture.database,
"reader_socket": capture.reader_socket,
"peers": capture.peers,
"capture_binding": capture.capture_binding,
"mode": match capture.mode { config::CaptureMode::Create => "create", config::CaptureMode::Recover => "recover" },
"bootstrap": capture.bootstrap,
});
std::fs::write(&path, serde_json::to_vec(&value).expect("capture JSON")).expect("write configuration");
let mut config = crate::config::Config::new(format!("127.0.0.1:{}", 19000 + capture.identity.node_id), Vec::new());
config.native_capture_config = Some(path);
config
}
async fn leader(runtime: &NativeCaptureRuntime) {
let mut metrics = runtime.owner().metrics();
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if metrics.borrow().current_leader == Some(runtime.owner().identity().node_id) {
break;
}
metrics.changed().await.expect("live owner metrics");
}
})
.await
.expect("real single-node election");
}
#[tokio::test]
async fn native_capture_startup_disabled() {
let directory = private_directory();
let config = crate::config::Config::new("127.0.0.1:19000", Vec::new());
let slot = ServerContextSlot::new();
assert!(start_capture(&config, &slot).await.expect("disabled startup").is_none());
assert!(slot.installed_capture_service().is_none());
assert_eq!(std::fs::read_dir(directory.path()).expect("directory").count(), 0);
}
#[tokio::test]
async fn native_capture_startup_create_recover() {
let directory = private_directory();
let mut capture = capture_config(directory.path(), 0);
let slot = ServerContextSlot::new();
let mut runtime = start_capture(&server_config(&capture), &slot)
.await
.expect("create")
.expect("enabled");
runtime.finish_startup("127.0.0.1:19000").await.expect("bootstrap");
leader(&runtime).await;
let result = runtime.owner().client_write(command(1)).await.expect("actual Raft write");
let client = SourceReaderClient::new(&capture.reader_socket).expect("shared source reader");
let before = client
.page(capture.identity.cluster_id, None, 1)
.await
.expect("committed page");
assert_eq!(before.identity, capture.identity);
assert_eq!(before.page.events.len(), 1);
assert_eq!(
before.page.events[0].captured.decision_log_id,
rustfs_capture_authority::LogId::from(result.log_id)
);
runtime
.owner()
.raft()
.trigger()
.snapshot()
.await
.expect("actual startup owner snapshot trigger");
runtime
.owner()
.raft()
.wait(Some(Duration::from_secs(10)))
.metrics(
|metrics| metrics.snapshot.is_some_and(|cut| cut.index >= result.log_id.index),
"startup create produces a real durable snapshot",
)
.await
.expect("NATIVE_CAPTURE_STARTUP_PROVENANCE_SUPPORTS_SNAPSHOT");
runtime.shutdown().await.expect("full shutdown");
drop(slot);
capture.mode = config::CaptureMode::Recover;
capture.bootstrap = false;
let slot = ServerContextSlot::new();
let mut runtime = start_capture(&server_config(&capture), &slot)
.await
.expect("recover")
.expect("enabled");
runtime
.finish_startup("127.0.0.1:19000")
.await
.expect("recover without bootstrap");
let client = SourceReaderClient::new(&capture.reader_socket).expect("recovered reader");
assert_eq!(
client
.page(capture.identity.cluster_id, None, 1)
.await
.expect("retained page"),
before
);
leader(&runtime).await;
runtime.owner().client_write(command(1)).await.expect("idempotent real write");
assert_eq!(
client
.page(capture.identity.cluster_id, None, 10)
.await
.expect("deduplicated page")
.page
.events
.len(),
1
);
runtime.shutdown().await.expect("recovered shutdown");
}
#[tokio::test]
async fn native_capture_startup_rejects_invalid_identity() {
let directory = private_directory();
let mut capture = capture_config(directory.path(), 0);
let slot = ServerContextSlot::new();
let mut runtime = start_capture(&server_config(&capture), &slot)
.await
.expect("create")
.expect("enabled");
runtime.finish_startup("127.0.0.1:19000").await.expect("bootstrap");
leader(&runtime).await;
runtime.owner().client_write(command(2)).await.expect("retained native event");
runtime.shutdown().await.expect("shutdown");
drop(slot);
let original_identity = capture.identity;
let stable = durable_state(&capture.database, original_identity).await;
capture.mode = config::CaptureMode::Recover;
capture.bootstrap = false;
let vote_before = Store::open(&capture.database, capture.identity)
.expect("retained identity")
.read_vote()
.expect("retained vote");
let pending_slot = ServerContextSlot::new();
let mut pending = start_capture(&server_config(&capture), &pending_slot)
.await
.expect("recover before audience publication")
.expect("enabled");
assert!(pending.owner.is_none(), "audience validation must precede actual Raft construction");
let defaults = openraft::Config::default();
tokio::time::sleep(Duration::from_millis(defaults.election_timeout_max + 2 * defaults.heartbeat_interval)).await;
use rustfs_protos::proto_gen::node_service::capture_raft_service_server::CaptureRaftService;
let pending_service = pending_slot.installed_capture_service().expect("installed early holder");
let error = pending_service
.vote(tonic::Request::new(rustfs_protos::proto_gen::node_service::CaptureRaftRequest {
body: bytes::Bytes::new(),
}))
.await
.expect_err("RPC admission must wait for actual Ping identity");
assert_eq!(error.code(), tonic::Code::Unavailable);
assert!(pending.finish_startup("127.0.0.1:19001").await.is_err());
pending.shutdown().await.expect("wrong audience recovery cleanup");
assert_eq!(durable_state(&capture.database, original_identity).await, stable);
assert_eq!(
Store::open(&capture.database, capture.identity)
.expect("immediate reopen")
.read_vote()
.expect("vote"),
vote_before
);
capture.identity.cluster_id = [7; 16];
let slot = ServerContextSlot::new();
assert!(
start_capture(&server_config(&capture), &slot).await.is_err(),
"STARTUP_RECOVER_REQUIRES_DURABLE_IDENTITY"
);
assert_eq!(durable_state(&capture.database, original_identity).await, stable);
assert!(slot.installed_capture_service().is_none());
capture.identity.cluster_id = [1; 16];
capture.capture_binding.binding_id = [7; 16];
assert!(
start_capture(&server_config(&capture), &slot).await.is_err(),
"STARTUP_RECOVER_REQUIRES_DURABLE_BINDING"
);
assert_eq!(durable_state(&capture.database, original_identity).await, stable);
assert!(slot.installed_capture_service().is_none());
capture.capture_binding.binding_id = [8; 16];
let foreign_listener = std::os::unix::net::UnixListener::bind(&capture.reader_socket).expect("foreign socket");
use std::os::unix::fs::MetadataExt;
let foreign_inode = std::fs::symlink_metadata(&capture.reader_socket)
.expect("foreign socket metadata")
.ino();
assert!(start_capture(&server_config(&capture), &slot).await.is_err(), "occupied reader socket");
assert_eq!(durable_state(&capture.database, original_identity).await, stable);
assert_eq!(
std::fs::symlink_metadata(&capture.reader_socket)
.expect("foreign socket preserved")
.ino(),
foreign_inode
);
assert!(slot.installed_capture_service().is_none());
drop(foreign_listener);
std::fs::remove_file(&capture.reader_socket).expect("remove our foreign socket");
capture.database = directory.path().join("missing.redb");
assert!(start_capture(&server_config(&capture), &slot).await.is_err());
assert!(!capture.database.exists());
capture.mode = config::CaptureMode::Create;
let config = server_config(&capture);
let config_path = config.native_capture_config.as_ref().expect("configuration path");
let mut invalid: serde_json::Value =
serde_json::from_slice(&std::fs::read(config_path).expect("configuration bytes")).expect("configuration JSON");
invalid["unexpected"] = serde_json::json!(true);
std::fs::write(config_path, serde_json::to_vec(&invalid).expect("unknown field JSON")).expect("write unknown field");
assert!(start_capture(&config, &slot).await.is_err(), "strict configuration rejects unknown field");
assert!(!capture.database.exists());
assert!(!capture.reader_socket.exists());
invalid.as_object_mut().expect("configuration object").remove("unexpected");
invalid["peers"]["0"]["rpc_uri"] = serde_json::json!("http://127.0.0.1:0");
std::fs::write(config_path, serde_json::to_vec(&invalid).expect("port zero JSON")).expect("write port zero");
assert!(start_capture(&config, &slot).await.is_err(), "fixed peer port required");
assert!(!capture.database.exists());
assert!(!capture.reader_socket.exists());
assert_eq!(durable_state(&directory.path().join("capture.redb"), original_identity).await, stable);
}
#[tokio::test]
async fn native_capture_startup_failure_drains_owner() {
let directory = private_directory();
let mut capture = capture_config(directory.path(), 0);
let slot = ServerContextSlot::new();
let mut runtime = start_capture(&server_config(&capture), &slot)
.await
.expect("create")
.expect("enabled");
runtime
.finish_startup("127.0.0.1:19000")
.await
.expect("actual owner started before later failure");
assert!(runtime.owner.is_some());
assert!(
runtime.finish_startup("127.0.0.1:19001").await.is_err(),
"startup must reject wrong Ping audience"
);
runtime.shutdown().await.expect("startup error cleanup");
drop(slot);
assert!(!capture.reader_socket.exists());
capture.mode = config::CaptureMode::Recover;
capture.bootstrap = false;
let slot = ServerContextSlot::new();
let runtime = start_capture(&server_config(&capture), &slot)
.await
.expect("immediate same-identity reopen")
.expect("enabled");
runtime.shutdown().await.expect("shutdown reopened runtime");
}
#[tokio::test]
async fn native_capture_startup_instance_isolation() {
let first_directory = private_directory();
let second_directory = private_directory();
let first_config = capture_config(first_directory.path(), 0);
let mut second_config = capture_config(second_directory.path(), 1);
second_config.identity.cluster_id = [6; 16];
let first = ServerContextSlot::new();
let second = ServerContextSlot::new();
let empty = ServerContextSlot::new();
assert!(first.installed_capture_service().is_none());
let mut a = start_capture(&server_config(&first_config), &first)
.await
.expect("first")
.expect("enabled");
let mut b = start_capture(&server_config(&second_config), &second)
.await
.expect("second")
.expect("enabled");
a.finish_startup("127.0.0.1:19000").await.expect("first real owner");
b.finish_startup("127.0.0.1:19001").await.expect("second real owner");
let a_service = first.installed_capture_service().expect("first service");
let b_service = second.installed_capture_service().expect("second service");
assert!(!Arc::ptr_eq(&a_service, &b_service));
assert!(empty.installed_capture_service().is_none());
let a_client = SourceReaderClient::new(&first_config.reader_socket).expect("first reader");
let b_client = SourceReaderClient::new(&second_config.reader_socket).expect("second reader");
assert_eq!(
a_client
.page(first_config.identity.cluster_id, None, 1)
.await
.expect("first page")
.identity,
first_config.identity
);
assert_eq!(
b_client
.page(second_config.identity.cluster_id, None, 1)
.await
.expect("second page")
.identity,
second_config.identity
);
assert!(a_client.page(second_config.identity.cluster_id, None, 1).await.is_err());
a.shutdown().await.expect("first shutdown");
let reopened = Store::open(&first_config.database, first_config.identity)
.expect("idle HTTP service holder must release real Store after shutdown");
drop(reopened);
use rustfs_protos::proto_gen::node_service::capture_raft_service_server::CaptureRaftService;
let error = a_service
.vote(tonic::Request::new(rustfs_protos::proto_gen::node_service::CaptureRaftRequest {
body: bytes::Bytes::new(),
}))
.await
.expect_err("old HTTP service is unmounted");
assert_eq!(error.code(), tonic::Code::Unavailable, "CAPTURE_IDLE_ROUTES_DO_NOT_RETAIN_STORE");
drop(a_service);
drop(b_service);
b.shutdown().await.expect("second shutdown");
}
#[tokio::test]
async fn native_capture_startup_defers_committed_replay_until_identity() {
use openraft::storage::{RaftLogStorage, RaftLogStorageExt, RaftStateMachine};
use rustfs_capture_authority::{CaptureRaftEntry, LogId, NativeRaftLogStore, NativeRaftStateMachine};
let directory = private_directory();
let mut capture = capture_config(directory.path(), 0);
let id = |index| LogId {
term: 1,
leader_node: 0,
index,
};
let mut store = Store::initialize(&capture.database, capture.identity).expect("fixture database");
store
.pin_capture_binding(capture.capture_binding.clone())
.expect("fixture binding");
store.enable_native_empty().expect("native mode");
store.promote_created_provenance().expect("native provenance");
let writer = NativeLogWriter::new(store);
let mut logs = NativeRaftLogStore::new(writer.clone());
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("real SM");
logs.save_vote(&openraft::Vote::new_committed(1, 0))
.await
.expect("actual durable leader vote");
let membership = CaptureRaftEntry {
log_id: id(0).into(),
payload: openraft::EntryPayload::Membership(openraft::Membership::new(
vec![std::collections::BTreeSet::from([0])],
capture.peers.clone(),
)),
};
logs.blocking_append(vec![membership.clone()])
.await
.expect("persist membership");
logs.save_committed(Some(id(0).into())).await.expect("commit membership");
sm.apply(vec![membership]).await.expect("apply membership prefix");
let pending = CaptureRaftEntry {
log_id: id(1).into(),
payload: openraft::EntryPayload::Normal(command(3)),
};
logs.blocking_append(vec![pending]).await.expect("persist pending Created");
logs.save_committed(Some(id(1).into()))
.await
.expect("persist commit before apply");
// Prepare the real committed-before-apply crash boundary, not a consensus result.
drop(sm);
drop(logs);
drop(writer);
let retained = Store::open(&capture.database, capture.identity).expect("inspect fixture");
let before_page = retained.created_events_page(None, 256).expect("initial source page");
let before_state = retained.native_application_state().expect("initial applied/membership");
assert_eq!(before_state.0, Some(id(0)));
assert!(before_page.events.is_empty());
drop(retained);
let before_durable = durable_state(&capture.database, capture.identity).await;
capture.mode = config::CaptureMode::Recover;
capture.bootstrap = false;
let slot = ServerContextSlot::new();
let mut runtime = start_capture(&server_config(&capture), &slot)
.await
.expect("prepare recovery")
.expect("enabled");
assert!(runtime.owner.is_none());
assert!(runtime.finish_startup("127.0.0.1:19001").await.is_err());
runtime.shutdown().await.expect("failed startup cleanup");
drop(slot);
assert_eq!(
durable_state(&capture.database, capture.identity).await,
before_durable,
"WRONG_AUDIENCE_MUST_NOT_START_COMMITTED_REPLAY"
);
let retained = Store::open(&capture.database, capture.identity).expect("inspect rejection");
assert_eq!(retained.created_events_page(None, 256).expect("retained page"), before_page);
assert_eq!(retained.native_application_state().expect("retained applied/membership"), before_state);
drop(retained);
let slot = ServerContextSlot::new();
let mut runtime = start_capture(&server_config(&capture), &slot)
.await
.expect("prepare valid recovery")
.expect("enabled");
runtime
.finish_startup("127.0.0.1:19000")
.await
.expect("validate then start actual owner");
let client = SourceReaderClient::new(&capture.reader_socket).expect("real reader");
let replayed = tokio::time::timeout(Duration::from_secs(10), async {
loop {
let page = client
.page(capture.identity.cluster_id, None, 256)
.await
.expect("source page");
if page.page.events.len() == 1 {
break page;
}
tokio::task::yield_now().await;
}
})
.await
.expect("actual committed replay completes");
assert_eq!(replayed.page.events[0].captured.decision_log_id, id(1));
runtime.shutdown().await.expect("shutdown valid recovery");
drop(slot);
let retained = Store::open(&capture.database, capture.identity).expect("inspect actual replay");
assert!(
retained
.native_application_state()
.expect("applied")
.0
.is_some_and(|value| value.index >= 1)
);
assert_eq!(retained.created_events_page(None, 256).expect("durable replay page"), replayed.page);
}
+72 -57
View File
@@ -1958,69 +1958,84 @@ async fn run(config: Config) -> Result<()> {
let endpoint_pools = init_startup_storage_foundation(&server_address, &config.volumes, &instance_ctx).await?;
let server_ctx = ServerContextSlot::with_instance_context(instance_ctx.clone());
let StartupHttpServers {
state_manager,
s3_shutdown_tx,
console_shutdown_tx,
} = init_startup_http_servers(&config, readiness.clone(), server_ctx.clone()).await?;
let mut capture_runtime = crate::startup_capture::start_capture(&config, &server_ctx).await?;
let result = async {
let StartupHttpServers {
state_manager,
s3_shutdown_tx,
console_shutdown_tx,
} = init_startup_http_servers(&config, readiness.clone(), server_ctx.clone()).await?;
let StartupStorageRuntime {
store,
shutdown_token: ctx,
} = init_startup_storage_runtime(server_addr, &endpoint_pools, readiness.clone(), instance_ctx).await?;
let StartupStorageRuntime {
store,
shutdown_token: ctx,
} = init_startup_storage_runtime(server_addr, &endpoint_pools, readiness.clone(), instance_ctx).await?;
#[cfg(feature = "e2e-test-hooks")]
if let Ok(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE") {
let nonce = uuid::Uuid::parse_str(&nonce).map_err(Error::other)?;
let release = std::path::PathBuf::from(
std::env::var_os("RUSTFS_E2E_STARTUP_CAS_RELEASE")
.ok_or_else(|| Error::other("startup CAS fixture requires a release path"))?,
);
if server_ctx.installed_object_store().is_some() {
return Err(Error::other("startup CAS gate reached an installed slot"));
}
let line = format!(
"RUSTFS_E2E_STARTUP_CAS {}\n",
serde_json::json!({
"kind": "gate", "nonce": nonce, "pid": std::process::id(), "slot_installed": false,
})
);
let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
tokio::time::timeout(std::time::Duration::from_secs(180), async {
while !tokio::fs::try_exists(&release).await? {
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
#[cfg(feature = "e2e-test-hooks")]
if let Ok(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE") {
let nonce = uuid::Uuid::parse_str(&nonce).map_err(Error::other)?;
let release = std::path::PathBuf::from(
std::env::var_os("RUSTFS_E2E_STARTUP_CAS_RELEASE")
.ok_or_else(|| Error::other("startup CAS fixture requires a release path"))?,
);
if server_ctx.installed_object_store().is_some() {
return Err(Error::other("startup CAS gate reached an installed slot"));
}
Ok::<_, Error>(())
let line = format!(
"RUSTFS_E2E_STARTUP_CAS {}\n",
serde_json::json!({
"kind": "gate", "nonce": nonce, "pid": std::process::id(), "slot_installed": false,
})
);
let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
tokio::time::timeout(std::time::Duration::from_secs(180), async {
while !tokio::fs::try_exists(&release).await? {
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
Ok::<_, Error>(())
})
.await
.map_err(|_| Error::other("startup CAS gate release timed out"))??;
}
if let Some(capture) = &mut capture_runtime {
let audience = crate::storage_api::server::http::try_current_local_node_name()
.ok_or_else(|| Error::other("native capture Ping audience unavailable"))?;
capture.finish_startup(&audience).await?;
}
let capacity_tasks = crate::capacity::capacity_integration::init_capacity_management_managed().await;
let service_runtime = init_startup_runtime_services(
&config,
endpoint_pools,
store.clone(),
ctx.clone(),
readiness.clone(),
state_manager.clone(),
server_ctx,
)
.await?;
run_startup_runtime_lifecycle(StartupRuntimeLifecycle {
server_address,
state_manager,
s3_shutdown_tx,
console_shutdown_tx,
capacity_tasks,
service_runtime,
capture_runtime: capture_runtime.take(),
store,
shutdown_token: ctx,
readiness,
})
.await
.map_err(|_| Error::other("startup CAS gate release timed out"))??;
}
let capacity_tasks = crate::capacity::capacity_integration::init_capacity_management_managed().await;
let service_runtime = init_startup_runtime_services(
&config,
endpoint_pools,
store.clone(),
ctx.clone(),
readiness.clone(),
state_manager.clone(),
server_ctx,
)
.await?;
run_startup_runtime_lifecycle(StartupRuntimeLifecycle {
server_address,
state_manager,
s3_shutdown_tx,
console_shutdown_tx,
capacity_tasks,
service_runtime,
store,
shutdown_token: ctx,
readiness,
})
.await
.await;
if let Some(capture) = capture_runtime {
capture.shutdown().await?;
}
result
}
#[cfg(test)]
+12 -2
View File
@@ -112,6 +112,7 @@ pub(crate) struct StartupRuntimeLifecycle {
pub(crate) console_shutdown_tx: Option<ShutdownHandle>,
pub(crate) capacity_tasks: Option<CapacityBackgroundTasks>,
pub(crate) service_runtime: StartupServiceRuntime,
pub(crate) capture_runtime: Option<crate::startup_capture::NativeCaptureRuntime>,
pub(crate) store: Arc<ECStore>,
pub(crate) shutdown_token: CancellationToken,
pub(crate) readiness: Arc<GlobalReadiness>,
@@ -125,6 +126,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
console_shutdown_tx,
capacity_tasks,
service_runtime,
capture_runtime,
store,
shutdown_token,
readiness,
@@ -149,14 +151,19 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
iam_bootstrap = ?iam_bootstrap,
"RustFS server ready"
);
publish_ready_for_iam_bootstrap(iam_bootstrap, readiness.as_ref(), Some(state_manager.as_ref())).await?;
if let Err(error) = publish_ready_for_iam_bootstrap(iam_bootstrap, readiness.as_ref(), Some(state_manager.as_ref())).await {
if let Some(capture) = capture_runtime {
capture.shutdown().await?;
}
return Err(error);
}
startup_runtime_sources::publish_init_time_now().await;
let event_notifier_reconciler = start_persisted_event_notifier_reconciler(store.clone(), shutdown_token.clone());
let scanner_cleanup = init_scanner_with_recovery(shutdown_token.clone(), store, enable_scanner).await;
let shutdown_signal = wait_for_shutdown().await;
run_startup_shutdown_sequence(
let shutdown_result = run_startup_shutdown_sequence(
&state_manager,
shutdown_signal,
s3_shutdown_tx,
@@ -164,6 +171,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
optional_runtimes,
capacity_tasks,
shutdown_token,
capture_runtime,
)
.await;
shutdown_connect_runtimes(heartbeat, inventory).await;
@@ -185,6 +193,8 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
);
}
shutdown_result?;
info!(
target: "rustfs::main::run",
event = EVENT_SERVER_SHUTDOWN_STATE,
+11 -1
View File
@@ -248,7 +248,11 @@ pub(crate) async fn run_startup_shutdown_sequence(
optional_runtimes: OptionalRuntimeServices,
capacity_tasks: Option<CapacityBackgroundTasks>,
ctx: CancellationToken,
) {
capture_runtime: Option<crate::startup_capture::NativeCaptureRuntime>,
) -> std::io::Result<()> {
if let Some(capture) = &capture_runtime {
capture.stop_admission();
}
ctx.cancel();
// Stop long-lived peer/disk background monitors while the runtime and tracing
@@ -463,6 +467,10 @@ pub(crate) async fn run_startup_shutdown_sequence(
console_shutdown_handle.shutdown().await;
}
shutdown_optional_runtime_services(optional_runtime_shutdowns).await;
let capture_result = match capture_runtime {
Some(capture) => capture.shutdown().await,
None => Ok(()),
};
// FTP, SFTP and WebDAV can start deletes too: wait for detached mutations
// only after every front end has stopped.
finish_detached_mutations_and_clear_unclean_shutdown_markers(
@@ -472,6 +480,7 @@ pub(crate) async fn run_startup_shutdown_sequence(
rustfs_heal::heal::clear_unclean_shutdown_markers(),
)
.await;
capture_result?;
state_manager.update(ServiceState::Stopped);
info!(
target: "rustfs::main::handle_shutdown",
@@ -481,6 +490,7 @@ pub(crate) async fn run_startup_shutdown_sequence(
state = "stopped",
"RustFS server stopped"
);
Ok(())
}
async fn run_embedded_runtime_cleanup(mut cleanup: EmbeddedRuntimeCleanup) {
+17 -2
View File
@@ -64,6 +64,13 @@ use std::sync::OnceLock;
use time::OffsetDateTime;
use url::{Url, form_urlencoded};
fn iam_authorization_error(err: IamError) -> ApiError {
match err {
IamError::IAMActionNotAllowed => ApiError::access_denied(),
other => ApiError::from(other),
}
}
const EVENT_OBJECT_TAG_AUTHORIZATION: &str = "object_tag_authorization";
const LOG_COMPONENT_ACCESS: &str = "storage_access";
const LOG_SUBSYSTEM_AUTHORIZATION: &str = "authorization";
@@ -1411,7 +1418,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
let allowed = iam_store
.try_eval_prepared(&prepared, &final_args)
.await
.map_err(ApiError::from)?;
.map_err(iam_authorization_error)?;
if !allowed
&& matches!(
action,
@@ -1433,7 +1440,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
if !iam_store
.try_eval_prepared(&prepared, &final_args)
.await
.map_err(ApiError::from)?
.map_err(iam_authorization_error)?
{
return Err(denial.deny("iam_explicit_deny", action));
}
@@ -3542,6 +3549,14 @@ mod tests {
use std::{collections::HashMap, sync::Arc};
use time::OffsetDateTime;
#[test]
fn terminal_iam_denial_is_access_denied_and_backend_failure_stays_internal() {
let denied = s3s::S3Error::from(super::iam_authorization_error(rustfs_iam::error::Error::IAMActionNotAllowed));
assert_eq!(denied.code(), &S3ErrorCode::AccessDenied);
let unavailable = s3s::S3Error::from(super::iam_authorization_error(rustfs_iam::error::Error::IamSysNotInitialized));
assert_eq!(unavailable.code(), &S3ErrorCode::InternalError);
}
struct UnreadyIam;
impl IamInterface for UnreadyIam {
+36 -1
View File
@@ -1741,6 +1741,36 @@ impl Node for NodeService {
async fn delete_paths(&self, request: Request<DeletePathsRequest>) -> Result<Response<DeletePathsResponse>, Status> {
self.handle_delete_paths(request).await
}
async fn native_preparation_begin(
&self,
request: Request<NativePreparationBeginRequest>,
) -> Result<Response<NativePreparationBeginResponse>, Status> {
self.handle_native_preparation_begin(request).await
}
async fn native_preparation_finalize(
&self,
request: Request<NativePreparationFinalizeRequest>,
) -> Result<Response<NativePreparationFinalizeResponse>, Status> {
self.handle_native_preparation_finalize(request).await
}
async fn native_preparation_probe(
&self,
request: Request<NativePreparationProbeRequest>,
) -> Result<Response<NativePreparationProbeResponse>, Status> {
self.handle_native_preparation_probe(request).await
}
async fn native_preparation_recover(
&self,
request: Request<NativePreparationRecoverRequest>,
) -> Result<Response<NativePreparationRecoverResponse>, Status> {
self.handle_native_preparation_recover(request).await
}
async fn native_preparation_locate(
&self,
request: Request<NativePreparationLocateRequest>,
) -> Result<Response<NativePreparationLocateResponse>, Status> {
self.handle_native_preparation_locate(request).await
}
async fn acquire_snapshot_lease(
&self,
request: Request<SnapshotLeaseRequest>,
@@ -3097,7 +3127,7 @@ mod tests {
use tonic::{Request, Response, Status};
use uuid::Uuid;
const DISK_MUTATION_RPC_METHODS: [&str; 24] = [
const DISK_MUTATION_RPC_METHODS: [&str; 29] = [
"renamedata",
"renamedataatincarnation",
"writemetadataatincarnation",
@@ -3122,6 +3152,11 @@ mod tests {
"acquiresnapshotlease",
"renewsnapshotlease",
"releasesnapshotlease",
"nativepreparationbegin",
"nativepreparationfinalize",
"nativepreparationprobe",
"nativepreparationrecover",
"nativepreparationlocate",
];
fn normalized_rpc_method(method: &str) -> String {
+413 -1
View File
@@ -19,7 +19,10 @@ use crate::storage::storage_api::rpc_consumer::node_service::{
validate_batch_read_version_item_count,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use crate::storage::storage_api::{PartTransactionAction, RenameDataResp, SnapshotLeaseToken, verify_tonic_mutation_body_digest};
use crate::storage::storage_api::{
PartTransactionAction, RenameDataResp, SnapshotLeaseToken, StorageError, sign_tonic_rpc_response_proof,
verify_tonic_mutation_body_digest,
};
use bytes::Bytes;
use rustfs_filemeta::FileInfo;
use rustfs_io_metrics::internode_metrics::{
@@ -118,6 +121,107 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024;
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
fn parse_native_uuid(bytes: &[u8], field: &str) -> std::result::Result<Uuid, StorageError> {
Uuid::from_slice(bytes)
.ok()
.filter(|id| !id.is_nil())
.ok_or_else(|| StorageError::other(format!("{field} must be a non-nil UUID")))
}
fn native_target(
disk: &str,
disk_id: &[u8],
store_id: &[u8],
preparation_id: &[u8],
bucket: &str,
key: &str,
bucket_incarnation_id: &[u8],
) -> std::result::Result<rustfs_ecstore::api::storage::NativePreparationTarget, StorageError> {
if disk.is_empty() {
return Err(StorageError::other("disk reference is required"));
}
Ok(rustfs_ecstore::api::storage::NativePreparationTarget {
disk_id: parse_native_uuid(disk_id, "disk_id")?,
store_id: parse_native_uuid(store_id, "store_id")?,
preparation: parse_native_uuid(preparation_id, "preparation_id")?,
bucket: bucket.to_owned(),
key: key.to_owned(),
bucket_incarnation: parse_native_uuid(bucket_incarnation_id, "bucket_incarnation_id")?,
})
}
fn native_receipt(
receipt: rustfs_ecstore::api::storage::NativePreparationReceipt,
) -> std::result::Result<NativePreparationReceipt, StorageError> {
let (data_blocks, parity_blocks, shard_index) = receipt.erasure_geometry();
Ok(NativePreparationReceipt {
disk_id: receipt.disk_id().as_bytes().to_vec().into(),
store_id: receipt.deployment_id().as_bytes().to_vec().into(),
deployment_id: receipt.deployment_id().as_bytes().to_vec().into(),
preparation_id: receipt.preparation_id().as_bytes().to_vec().into(),
version_id: receipt.version_id().as_bytes().to_vec().into(),
bucket: receipt.bucket().to_owned(),
key: receipt.key().to_owned(),
bucket_incarnation_id: receipt.bucket_incarnation().as_bytes().to_vec().into(),
object_digest: receipt.object_digest().to_vec().into(),
object_length: receipt.object_length(),
set_digest: receipt.set_digest().to_vec().into(),
distribution_digest: receipt.distribution_digest().to_vec().into(),
metadata_digest: receipt.metadata_digest().to_vec().into(),
shard_digest: receipt.shard_digest().to_vec().into(),
shard_length: receipt.shard_length(),
data_blocks: u32::try_from(data_blocks).map_err(|_| StorageError::other("data_blocks exceeds protocol range"))?,
parity_blocks: u32::try_from(parity_blocks).map_err(|_| StorageError::other("parity_blocks exceeds protocol range"))?,
shard_index: u32::try_from(shard_index).map_err(|_| StorageError::other("shard_index exceeds protocol range"))?,
})
}
fn native_error(error: StorageError) -> Option<Error> {
let error = error.narrow_to_disk().unwrap_or_else(DiskError::other);
// The generic I/O wire code loses ErrorKind; corruption must remain fatal to quorum callers.
let error = match error {
DiskError::Io(ref error) if error.kind() == std::io::ErrorKind::InvalidData => DiskError::FileCorrupt,
error => error,
};
Some(error.into())
}
fn validate_native_metadata_length(
metadata: &[u8],
bucket: &str,
key: &str,
version: Uuid,
object_length: u64,
) -> std::result::Result<(), StorageError> {
let mut file_meta = rustfs_filemeta::FileMeta::new();
let consumed = file_meta
.unmarshal_msg(metadata)
.map_err(|err| StorageError::other(format!("decode native preparation metadata: {err}")))?;
if usize::try_from(consumed).map_err(|_| StorageError::other("native preparation metadata length is invalid"))?
!= metadata.len()
{
return Err(StorageError::other("native preparation metadata has trailing bytes"));
}
let (_, version_meta) = file_meta
.find_version(Some(version))
.map_err(|err| StorageError::other(format!("native preparation version is absent from metadata: {err}")))?;
let info = version_meta
.into_fileinfo(bucket, key, true)
.map_err(|err| StorageError::other(format!("decode native preparation file info: {err}")))?;
if u64::try_from(info.size).map_err(|_| StorageError::other("native preparation object length is invalid"))? != object_length
{
return Err(StorageError::other("object_length does not match persisted metadata"));
}
Ok(())
}
fn sign_native_response(body: std::result::Result<Vec<u8>, std::num::TryFromIntError>) -> std::result::Result<Bytes, Status> {
let body = body.map_err(|_| Status::internal("native preparation response length cannot be represented"))?;
sign_tonic_rpc_response_proof(&body)
.map(Bytes::from)
.map_err(|_| Status::unavailable("native preparation response proof is unavailable"))
}
fn snapshot_lease_response(result: Result<SnapshotLeaseToken, DiskError>) -> Response<SnapshotLeaseResponse> {
match result {
Ok(token) => Response::new(SnapshotLeaseResponse {
@@ -378,6 +482,314 @@ fn encode_rename_data_response_payloads(
}
impl NodeService {
pub(super) async fn handle_native_preparation_begin(
&self,
request: Request<NativePreparationBeginRequest>,
) -> Result<Response<NativePreparationBeginResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_native_preparation_begin_request_body(request.get_ref()),
"native_preparation_begin",
)?;
let local_target = self.local_mutation_target();
let request = request.into_inner();
let result = async {
let target = native_target(
&request.disk,
&request.disk_id,
&request.store_id,
&request.preparation_id,
&request.bucket,
&request.key,
&request.bucket_incarnation_id,
)?;
let LocalMutationTarget::Ready(store) = local_target else {
return Err(StorageError::other("native preparation requires a ready storage instance"));
};
store.begin_native_preparation_on_disk(&target).await
}
.await;
let mut response = NativePreparationBeginResponse {
success: result.is_ok(),
error: result.err().and_then(native_error),
response_proof: Bytes::new(),
};
response.response_proof =
sign_native_response(rustfs_protos::canonical_native_preparation_begin_response_body(&request, &response))?;
Ok(Response::new(response))
}
pub(super) async fn handle_native_preparation_finalize(
&self,
request: Request<NativePreparationFinalizeRequest>,
) -> Result<Response<NativePreparationFinalizeResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_native_preparation_finalize_request_body(request.get_ref()),
"native_preparation_finalize",
)?;
let local_target = self.local_mutation_target();
let request = request.into_inner();
let result = async {
let target = native_target(
&request.disk,
&request.disk_id,
&request.store_id,
&request.preparation_id,
&request.bucket,
&request.key,
&request.bucket_incarnation_id,
)?;
let LocalMutationTarget::Ready(store) = local_target else {
return Err(StorageError::other("native preparation requires a ready storage instance"));
};
if request.metadata.len() > rustfs_protos::NATIVE_PREPARATION_METADATA_MAX_BYTES {
return Err(StorageError::other("native preparation metadata exceeds size limit"));
}
let object_digest: [u8; 32] = request
.object_digest
.as_ref()
.try_into()
.map_err(|_| StorageError::other("object_digest must contain 32 bytes"))?;
let version = parse_native_uuid(&request.version_id, "version_id")?;
validate_native_metadata_length(&request.metadata, &target.bucket, &target.key, version, request.object_length)?;
let receipt = store
.finalize_native_preparation_on_disk(rustfs_ecstore::api::storage::NativePreparationFinalizeRequest {
target,
version,
object_digest,
metadata: request.metadata.clone(),
})
.await?;
native_receipt(receipt)
}
.await;
let (success, receipt, error) = match result {
Ok(receipt) => (true, Some(receipt), None),
Err(error) => (false, None, native_error(error)),
};
let mut response = NativePreparationFinalizeResponse {
success,
receipt,
error,
response_proof: Bytes::new(),
};
response.response_proof =
sign_native_response(rustfs_protos::canonical_native_preparation_finalize_response_body(&request, &response))?;
Ok(Response::new(response))
}
pub(super) async fn handle_native_preparation_probe(
&self,
request: Request<NativePreparationProbeRequest>,
) -> Result<Response<NativePreparationProbeResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_native_preparation_probe_request_body(request.get_ref()),
"native_preparation_probe",
)?;
let local_target = self.local_mutation_target();
let request = request.into_inner();
let result = async {
let target = native_target(
&request.disk,
&request.disk_id,
&request.store_id,
&request.preparation_id,
&request.bucket,
&request.key,
&request.bucket_incarnation_id,
)?;
let LocalMutationTarget::Ready(store) = local_target else {
return Err(StorageError::other("native preparation requires a ready storage instance"));
};
store.probe_native_preparation_on_disk(&target).await
}
.await;
let (
success,
exists,
prepared,
version_id,
object_digest,
object_length,
set_digest,
distribution_digest,
metadata,
error,
) = match result {
Ok(rustfs_ecstore::api::storage::NativePreparationProbe::Absent) => (
true,
false,
false,
Bytes::new(),
Bytes::new(),
0,
Bytes::new(),
Bytes::new(),
Bytes::new(),
None,
),
Ok(rustfs_ecstore::api::storage::NativePreparationProbe::Existing {
version,
prepared,
object_digest,
object_length,
set_digest,
distribution_digest,
metadata,
}) => (
true,
true,
prepared,
version.as_bytes().to_vec().into(),
object_digest.to_vec().into(),
object_length,
set_digest.to_vec().into(),
distribution_digest.to_vec().into(),
metadata.unwrap_or_default(),
None,
),
Err(error) => (
false,
false,
false,
Bytes::new(),
Bytes::new(),
0,
Bytes::new(),
Bytes::new(),
Bytes::new(),
native_error(error),
),
};
let mut response = NativePreparationProbeResponse {
success,
exists,
prepared,
metadata,
error,
response_proof: Bytes::new(),
version_id,
object_digest,
object_length,
set_digest,
distribution_digest,
};
response.response_proof =
sign_native_response(rustfs_protos::canonical_native_preparation_probe_response_body(&request, &response))?;
Ok(Response::new(response))
}
pub(super) async fn handle_native_preparation_recover(
&self,
request: Request<NativePreparationRecoverRequest>,
) -> Result<Response<NativePreparationRecoverResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_native_preparation_recover_request_body(request.get_ref()),
"native_preparation_recover",
)?;
let local_target = self.local_mutation_target();
let request = request.into_inner();
let result = async {
let target = native_target(
&request.disk,
&request.disk_id,
&request.store_id,
&request.preparation_id,
&request.bucket,
&request.key,
&request.bucket_incarnation_id,
)?;
let LocalMutationTarget::Ready(store) = local_target else {
return Err(StorageError::other("native preparation requires a ready storage instance"));
};
let recovered = store.recover_native_preparation_on_disk(&target).await?;
Ok::<_, StorageError>((native_receipt(recovered.receipt)?, recovered.metadata))
}
.await;
let (success, receipt, metadata, error) = match result {
Ok((receipt, metadata)) => (true, Some(receipt), metadata, None),
Err(error) => (false, None, Bytes::new(), native_error(error)),
};
let mut response = NativePreparationRecoverResponse {
success,
receipt,
metadata,
error,
response_proof: Bytes::new(),
};
response.response_proof =
sign_native_response(rustfs_protos::canonical_native_preparation_recover_response_body(&request, &response))?;
Ok(Response::new(response))
}
pub(super) async fn handle_native_preparation_locate(
&self,
request: Request<NativePreparationLocateRequest>,
) -> Result<Response<NativePreparationLocateResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_native_preparation_locate_request_body(request.get_ref()),
"native_preparation_locate",
)?;
let local_target = self.local_mutation_target();
let request = request.into_inner();
let result = async {
if request.disk.is_empty() {
return Err(StorageError::other("disk reference is required"));
}
let disk_id = parse_native_uuid(&request.disk_id, "disk_id")?;
let store_id = parse_native_uuid(&request.store_id, "store_id")?;
let preparation_id = parse_native_uuid(&request.preparation_id, "preparation_id")?;
let LocalMutationTarget::Ready(store) = local_target else {
return Err(StorageError::other("native preparation requires a ready storage instance"));
};
store
.locate_native_preparation_on_disk(disk_id, store_id, preparation_id)
.await
}
.await;
let (success, found, bucket, key, incarnation, metadata, prepared, error) = match result {
Ok(Some(description)) => (
true,
true,
description.bucket,
description.key,
description.incarnation.as_bytes().to_vec().into(),
description.metadata,
description.prepared,
None,
),
Ok(None) => (true, false, String::new(), String::new(), Bytes::new(), Bytes::new(), false, None),
Err(error) => (
false,
false,
String::new(),
String::new(),
Bytes::new(),
Bytes::new(),
false,
native_error(error),
),
};
let mut response = NativePreparationLocateResponse {
success,
found,
bucket,
key,
bucket_incarnation_id: incarnation,
metadata,
prepared,
error,
response_proof: Bytes::new(),
};
response.response_proof =
sign_native_response(rustfs_protos::canonical_native_preparation_locate_response_body(&request, &response))?;
Ok(Response::new(response))
}
pub(super) async fn handle_acquire_snapshot_lease(
&self,
request: Request<SnapshotLeaseRequest>,