mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-27 14:27:10 +00:00
Compare commits
218 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1f8d013b8d | |||
| b8858d0c70 | |||
| 4fd444852c | |||
| 05f84793a3 | |||
| 2fa9a433ad | |||
| 08811ec243 | |||
| 0f89e9b8b8 | |||
| de8b465c41 | |||
| 03f77844b7 | |||
| b44c76db52 | |||
| d0096af742 | |||
| 35e07b44f4 | |||
| 7d91176cea | |||
| 001d935569 | |||
| cc157adca8 | |||
| e0917fb617 | |||
| 6a30694765 | |||
| 991d647c78 | |||
| aafa85c6e5 | |||
| 01f5aa9c3a | |||
| e27bfb2006 | |||
| f095c6a476 | |||
| a223e74c83 | |||
| 1d1964215d | |||
| 5b23462652 | |||
| 02f6d87201 | |||
| 37a54d3ff2 | |||
| d1aaaee9bf | |||
| bebd5d0631 | |||
| 0de18a3577 | |||
| 04756b0a93 | |||
| 6fc62edcf7 | |||
| 68d1790b74 | |||
| 6fde372f5b | |||
| 503a852214 | |||
| ab7a8c400f | |||
| 85663e9647 | |||
| 61e4cfbc2a | |||
| 675722e5f8 | |||
| f6dfe86c58 | |||
| 3c9b9589f2 | |||
| a5ad283c8c | |||
| dcdd53ccf3 | |||
| bdd7efc96d | |||
| 9fbf5d0553 | |||
| e7bebb0fc5 | |||
| cacdaa65a0 | |||
| efe2255ccf | |||
| f4b694ed19 | |||
| aa95a32ea2 | |||
| 3f3e892d86 | |||
| 874d2babcc | |||
| 7c85783a08 | |||
| 541fbbe74a | |||
| 08ba7a7976 | |||
| 81f214c99a | |||
| dff05785de | |||
| cf2e3af4d9 | |||
| e0f0628bc1 | |||
| 6de3034e36 | |||
| ae259e127d | |||
| 8233e3fe4c | |||
| f88b7b6ba5 | |||
| 0307c9a7e2 | |||
| 6a8d0aba58 | |||
| ee6c2afe43 | |||
| 9ca8bc3ca6 | |||
| 3678703a98 | |||
| fe798c13e7 | |||
| 24b77349aa | |||
| 8c55396bfe | |||
| 5be8badb20 | |||
| 46daaadbb3 | |||
| f3b6c4df6b | |||
| 7183849227 | |||
| 942ce79c4e | |||
| fec1262f1a | |||
| 5eb46cb29d | |||
| 2cd904d0e7 | |||
| 61eec6da18 | |||
| 5b8cd2a5dc | |||
| d7824049b2 | |||
| d46e4c54a6 | |||
| 3ca0b5f35e | |||
| 00b5ddc0ad | |||
| ad4c2bc9b1 | |||
| ef5fe69d52 | |||
| ba6f37e58d | |||
| 31eb2e5a4a | |||
| 2f3654793d | |||
| ce09e7817c | |||
| 31dc8a0fea | |||
| d243a4a29e | |||
| 1048374cbd | |||
| de32dece41 | |||
| f79f318ec4 | |||
| 60fe68a04a | |||
| 498be1fa53 | |||
| b798d291d7 | |||
| 57c4d4c719 | |||
| 560b083278 | |||
| 36afe8bb57 | |||
| 57963574b7 | |||
| c56a597374 | |||
| 5354e6dccf | |||
| 7608c5af11 | |||
| c926e10de0 | |||
| 972dafccde | |||
| 5116bb2bbb | |||
| 08d6e4343d | |||
| 7d3a6acc57 | |||
| aabb26b519 | |||
| 163e6064da | |||
| c7a9b76543 | |||
| 7cc4ce5447 | |||
| 44fe49e5fa | |||
| dde2c50309 | |||
| d5a8f022bd | |||
| 150d349ade | |||
| 38056f3783 | |||
| 0b40e35e76 | |||
| 458b144ac7 | |||
| d58b80d54e | |||
| f8d0174fa7 | |||
| a57cc0d44c | |||
| 8006cb7eae | |||
| 8d40a995a3 | |||
| a451b53bcc | |||
| a6127c8fde | |||
| f346c13e5b | |||
| 2634e829ca | |||
| dc15444256 | |||
| ebb19907a9 | |||
| d29babf251 | |||
| d8a9522919 | |||
| 60dc84bf0e | |||
| 061e89ed2b | |||
| 3065142735 | |||
| c6a8e30f5a | |||
| f51b52d7e9 | |||
| c54160c22c | |||
| 1f67b65ab8 | |||
| a7806e65f0 | |||
| 6a3678997a | |||
| 64ea8b8e09 | |||
| 7773aac0fd | |||
| 8763f87ca7 | |||
| 9bbac59977 | |||
| 3bd57517ff | |||
| 8ee78c6891 | |||
| 21c594e625 | |||
| 748095280a | |||
| 960195458f | |||
| de58fdfc7d | |||
| 869d5ec24b | |||
| 19902438a6 | |||
| bd244d13f5 | |||
| 38395d35ec | |||
| 68064b5ad9 | |||
| c1da0c361b | |||
| cb7626c1f1 | |||
| 8186a7175c | |||
| c9e1446126 | |||
| 11d21ec6f4 | |||
| 974ab6f785 | |||
| c3f4cdf27c | |||
| e82d7c107a | |||
| 9d2d6a8e46 | |||
| ff2cbc3c47 | |||
| d65ef3e29a | |||
| 0a2f58c7a9 | |||
| 4d97501327 | |||
| 3b2af12c64 | |||
| c73ddde848 | |||
| ef72a37fd7 | |||
| e46cf750dd | |||
| fe955fa3ee | |||
| 64578c067e | |||
| 78715f0921 | |||
| 3284aafd73 | |||
| 9286442f87 | |||
| b8aad1c168 | |||
| 5fcb229760 | |||
| 94baef7462 | |||
| 60d90f085a | |||
| dfa5d50565 | |||
| a902886737 | |||
| 8e53b87421 | |||
| f7631e91b1 | |||
| f9bc4e921f | |||
| 781584307a | |||
| 59d641a5fe | |||
| c618b57870 | |||
| bbe1d54098 | |||
| 9e8ea4dcdb | |||
| 63cf8e2f2f | |||
| b7fe40f4b3 | |||
| 5b872b8e56 | |||
| ca5be1ef95 | |||
| 9373e8fa3e | |||
| cb32965522 | |||
| fb6cf49a67 | |||
| 8c97049e68 | |||
| 17d4fe27cc | |||
| fb0c7972b2 | |||
| eba7967755 | |||
| 451c1c31ad | |||
| 8780dd670e | |||
| f29bf2c902 | |||
| fd03436090 | |||
| 1f1e7b1c97 | |||
| 45a952609e | |||
| 323190a95a | |||
| 38cdda9292 | |||
| 85c84fa73d | |||
| 51402087c5 | |||
| a994984fe2 | |||
| 48eb0e402c |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=cf31ed27e4e6bf42da5bf81f248d9d2619ac8e8615186cdacb30bd63996d7607
|
||||
sha256-darwin=cf31ed27e4e6bf42da5bf81f248d9d2619ac8e8615186cdacb30bd63996d7607
|
||||
sha256-linux=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
|
||||
sha256-darwin=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
|
||||
|
||||
@@ -25,6 +25,8 @@ on:
|
||||
- "crates/ecstore/src/object_api/write_commit_context.rs"
|
||||
- "crates/ecstore/src/bucket/utils.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "crates/storage-api/src/bucket.rs"
|
||||
- "crates/storage-api/Cargo.toml"
|
||||
- "crates/utils/src/dunce.rs"
|
||||
- "crates/utils/src/os/**"
|
||||
- "crates/utils/Cargo.toml"
|
||||
@@ -48,6 +50,8 @@ on:
|
||||
- "crates/ecstore/src/object_api/write_commit_context.rs"
|
||||
- "crates/ecstore/src/bucket/utils.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "crates/storage-api/src/bucket.rs"
|
||||
- "crates/storage-api/Cargo.toml"
|
||||
- "crates/utils/src/dunce.rs"
|
||||
- "crates/utils/src/os/**"
|
||||
- "crates/utils/Cargo.toml"
|
||||
@@ -140,6 +144,17 @@ jobs:
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-ecstore --lib bucket::utils:: -- --nocapture"
|
||||
|
||||
- name: Test artifact prefix Windows file-name rules
|
||||
shell: pwsh
|
||||
run: |
|
||||
$testName = 'bucket::tests::artifact_prefix_preserves_host_file_name_rules'
|
||||
cargo test --locked -p rustfs-storage-api --lib $testName -- --exact --nocapture --color never 2>&1 |
|
||||
Tee-Object -Variable testOutput
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
if (($testOutput -join "`n") -notmatch 'test result: ok\. 1 passed; 0 failed; 0 ignored;') {
|
||||
throw 'Expected exactly one passing artifact prefix test on Windows'
|
||||
}
|
||||
|
||||
- name: Test Windows file-name rules
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
|
||||
|
||||
Generated
+2
@@ -10298,6 +10298,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10687,6 +10688,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
|
||||
@@ -200,8 +200,8 @@ python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run
|
||||
python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
# Distributed 4-node 4-disk lane uses the fault-test hooks server; the
|
||||
# scanner_heal_topology_ fault cases fail closed without it.
|
||||
# Distributed 4-node 4-disk lane includes physical-worker fault injection and
|
||||
# scanner_heal_topology_ cases; both require the fault-test hooks server.
|
||||
# Upgrade cases require RUSTFS_UPGRADE_SOURCE_BINARY and fail closed without it.
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, payload_for, s3_probe_client, unique_bucket, wait_for_ready,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier};
|
||||
use serde_json::json;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const PUBLIC_INTERNAL_ERROR: &str = "We encountered an internal error, please try again.";
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub(super) struct PhysicalEntry {
|
||||
identity: (u64, u64),
|
||||
changed: (i64, i64),
|
||||
modified: (i64, i64),
|
||||
bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
// Include directory identity/ctime as well as file bytes. Creating and removing
|
||||
// a temporary marker/rollback file cannot hide behind a restored final xl.meta.
|
||||
// Reads change atime, which is deliberately excluded.
|
||||
pub(super) fn physical_snapshot(dist: &DistCluster, bucket: &str) -> TestResult<BTreeMap<(usize, PathBuf), PhysicalEntry>> {
|
||||
let mut snapshot = BTreeMap::new();
|
||||
for (node, instance) in dist.cluster.nodes.iter().enumerate() {
|
||||
let root = Path::new(&instance.data_dirs[0]).join(bucket);
|
||||
for entry in walkdir::WalkDir::new(&root) {
|
||||
let entry = entry?;
|
||||
let metadata = std::fs::symlink_metadata(entry.path())?;
|
||||
assert!(!metadata.file_type().is_symlink(), "unexpected fixture symlink");
|
||||
snapshot.insert(
|
||||
(node, entry.path().strip_prefix(&root)?.to_path_buf()),
|
||||
PhysicalEntry {
|
||||
identity: (metadata.dev(), metadata.ino()),
|
||||
changed: (metadata.ctime(), metadata.ctime_nsec()),
|
||||
modified: (metadata.mtime(), metadata.mtime_nsec()),
|
||||
bytes: if metadata.is_dir() {
|
||||
None
|
||||
} else {
|
||||
Some(std::fs::read(entry.path())?)
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
struct Version {
|
||||
key: String,
|
||||
id: String,
|
||||
body: Vec<u8>,
|
||||
manifests: Vec<VersionShardCensus>,
|
||||
}
|
||||
|
||||
impl Version {
|
||||
async fn put(dist: &DistCluster, bucket: &str, key: &str, seed: &str) -> TestResult<Self> {
|
||||
let body = payload_for(seed, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let id = output.version_id().ok_or("fixture PUT omitted version ID")?.to_owned();
|
||||
let manifests = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, key, Some(&id)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
assert!(
|
||||
manifests
|
||||
.iter()
|
||||
.all(|manifest| manifest.is_complete() && !manifest.expected_part_numbers.is_empty())
|
||||
);
|
||||
Ok(Self {
|
||||
key: key.to_owned(),
|
||||
id,
|
||||
body,
|
||||
manifests,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn target(key: &str, version: Option<&str>) -> TestResult<ObjectIdentifier> {
|
||||
Ok(ObjectIdentifier::builder()
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.build()?)
|
||||
}
|
||||
|
||||
async fn assert_fixture(
|
||||
dist: &DistCluster,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
versions: &[Version],
|
||||
marker: (&str, &str),
|
||||
) -> TestResult {
|
||||
let expected: BTreeSet<_> = versions
|
||||
.iter()
|
||||
.map(|version| (version.key.clone(), version.id.clone()))
|
||||
.collect();
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node)?;
|
||||
let listed = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.list_object_versions().bucket(bucket).prefix(prefix).send(),
|
||||
)
|
||||
.await??;
|
||||
let actual: BTreeSet<_> = listed
|
||||
.versions()
|
||||
.iter()
|
||||
.map(|version| {
|
||||
(
|
||||
version.key().unwrap_or_default().to_owned(),
|
||||
version.version_id().unwrap_or_default().to_owned(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(listed.versions().len(), versions.len());
|
||||
assert_eq!(actual, expected, "delete changed exact data versions on node {node}");
|
||||
assert_eq!(listed.delete_markers().len(), 1);
|
||||
assert_eq!(listed.delete_markers()[0].key(), Some(marker.0));
|
||||
assert_eq!(listed.delete_markers()[0].version_id(), Some(marker.1));
|
||||
assert_eq!(listed.delete_markers()[0].is_latest(), Some(true));
|
||||
for version in versions {
|
||||
assert_exact_version(&client, bucket, &version.key, &version.id, &version.body).await?;
|
||||
let disk = Path::new(&dist.cluster.nodes[node].data_dirs[0]);
|
||||
assert!(
|
||||
census_object_version_on_disk(disk, bucket, &version.key, Some(&version.id))?
|
||||
.matches_manifest(&version.manifests[node])
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_denied(dist: &DistCluster, node: usize, bucket: &str, key: &str, version: Option<&str>) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
// One attempt: a later retry must not conceal a first local mutation.
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
let error = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.send(),
|
||||
)
|
||||
.await?
|
||||
.expect_err("ordinary deletion succeeded in a managed bucket");
|
||||
let service = error
|
||||
.as_service_error()
|
||||
.ok_or("delete failed without an S3 service rejection")?;
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(500));
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(PUBLIC_INTERNAL_ERROR));
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"first delete through node {node} wrote or rolled back managed data"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_batch_denied(dist: &DistCluster, node: usize, bucket: &str, objects: &[ObjectIdentifier]) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
let response = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(Delete::builder().set_objects(Some(objects.to_vec())).quiet(false).build()?)
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
assert!(response.deleted().is_empty(), "managed batch reported a successful delete");
|
||||
assert_eq!(response.errors().len(), objects.len(), "batch must reject every requested occurrence");
|
||||
for (error, requested) in response.errors().iter().zip(objects) {
|
||||
assert_eq!(error.key(), Some(requested.key()));
|
||||
assert_eq!(error.version_id(), requested.version_id());
|
||||
assert_eq!(error.code(), Some("InternalError"));
|
||||
assert_eq!(error.message(), Some(PUBLIC_INTERNAL_ERROR));
|
||||
}
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"first batch through node {node} wrote or rolled back managed data"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn copy_existing_object(source: &Path, destination: &Path) -> TestResult {
|
||||
assert!(!destination.exists());
|
||||
for entry in walkdir::WalkDir::new(source) {
|
||||
let entry = entry?;
|
||||
let target = destination.join(entry.path().strip_prefix(source)?);
|
||||
if entry.file_type().is_dir() {
|
||||
std::fs::create_dir_all(&target)?;
|
||||
} else {
|
||||
assert!(entry.file_type().is_file());
|
||||
std::fs::copy(entry.path(), target)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ordinary_delete_control(dist: &DistCluster, bucket: &str) -> TestResult {
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
let key = "ordinary-observer-control.bin";
|
||||
let version = Version::put(dist, bucket, key, key).await?;
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let deleted = client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(deleted.delete_marker(), Some(true));
|
||||
let marker = deleted.version_id().ok_or("ordinary delete omitted marker ID")?;
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? != before,
|
||||
"physical observer missed real marker creation"
|
||||
);
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id(marker)
|
||||
.send()
|
||||
.await?;
|
||||
// The logical version inventory is restored, but ctime/identity still
|
||||
// expose the two writes. This checks the observer used by managed negatives.
|
||||
let listed = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
|
||||
assert_eq!(listed.versions().len(), 1);
|
||||
assert_eq!(listed.versions()[0].version_id(), Some(version.id.as_str()));
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? != before,
|
||||
"marker round-trip hid transient physical writes"
|
||||
);
|
||||
assert_exact_version(&client, bucket, key, &version.id, &version.body).await?;
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id(&version.id)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(key)
|
||||
.send()
|
||||
.await?
|
||||
.versions()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let current = Version::put(dist, bucket, "ordinary-batch-current.bin", "current").await?;
|
||||
let exact = Version::put(dist, bucket, "ordinary-batch-exact.bin", "exact").await?;
|
||||
let response = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(target(¤t.key, None)?)
|
||||
.objects(target(&exact.key, Some(&exact.id))?)
|
||||
.objects(target(&exact.key, Some(&exact.id))?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(response.errors().is_empty());
|
||||
assert_eq!(response.deleted().len(), 3);
|
||||
let listed = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(¤t.key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 1);
|
||||
assert_eq!(listed.delete_markers().len(), 1);
|
||||
assert!(
|
||||
client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(&exact.key)
|
||||
.send()
|
||||
.await?
|
||||
.versions()
|
||||
.is_empty()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_delete_entrypoints_preserve_versions_and_physical_state() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (identity, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = identity["bucket"]
|
||||
.as_str()
|
||||
.ok_or("managed receipt omitted bucket")?
|
||||
.to_owned();
|
||||
let prefix = identity["attempt_prefix"]
|
||||
.as_str()
|
||||
.ok_or("managed receipt omitted prefix")?
|
||||
.to_owned();
|
||||
let key = format!("{prefix}native.bin");
|
||||
let mut versions = vec![
|
||||
Version::put(&dist, &bucket, &key, "native-old").await?,
|
||||
Version::put(&dist, &bucket, &key, "native-new").await?,
|
||||
];
|
||||
let ordinary = unique_bucket("delete-control");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary).await?;
|
||||
|
||||
// A new managed bucket cannot legitimately create a marker. Model existing
|
||||
// persisted state from before adoption/upgrade using a real S3-generated
|
||||
// ordinary object+marker, copied disk-for-disk in the same erasure set.
|
||||
// Only this object's files move; bucket authority/configuration is untouched.
|
||||
let marker_key = format!("{prefix}existing-marker.bin");
|
||||
versions.push(Version::put(&dist, &ordinary, &marker_key, "existing-data").await?);
|
||||
let output = dist
|
||||
.client(2)?
|
||||
.delete_object()
|
||||
.bucket(&ordinary)
|
||||
.key(&marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(output.delete_marker(), Some(true));
|
||||
let marker = output.version_id().ok_or("ordinary fixture omitted marker ID")?.to_owned();
|
||||
dist.cluster.stop();
|
||||
for node in &dist.cluster.nodes {
|
||||
let disk = Path::new(&node.data_dirs[0]);
|
||||
copy_existing_object(&disk.join(&ordinary).join(&marker_key), &disk.join(&bucket).join(&marker_key))?;
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
|
||||
ordinary_delete_control(&dist, &ordinary).await?;
|
||||
let batch = vec![
|
||||
target(&key, None)?,
|
||||
target(&key, Some(&versions[0].id))?,
|
||||
target(&key, Some(&versions[1].id))?,
|
||||
target(&marker_key, Some(&marker))?,
|
||||
target(&key, Some(&versions[0].id))?,
|
||||
target(&marker_key, Some(&versions[2].id))?,
|
||||
];
|
||||
for phase in ["open", "sealed", "restarted"] {
|
||||
if phase == "sealed" {
|
||||
confirm_seal(&dist, &admin, &json!({"version": 1, "target": identity}).to_string(), &identity).await?;
|
||||
} else if phase == "restarted" {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_single_denied(&dist, node, &bucket, &key, None).await?;
|
||||
assert_single_denied(&dist, node, &bucket, &key, Some(&versions[0].id)).await?;
|
||||
assert_single_denied(&dist, node, &bucket, &marker_key, Some(&marker)).await?;
|
||||
assert_batch_denied(&dist, node, &bucket, &batch).await?;
|
||||
}
|
||||
assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_delete_test::physical_snapshot;
|
||||
use super::artifact_seal_test::{ArtifactAdmin, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, wait_for_ready};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
signed_s3_request,
|
||||
};
|
||||
use futures::{StreamExt as _, TryStreamExt as _};
|
||||
use http::{Method, StatusCode};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::{collections::BTreeSet, time::Duration};
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
const ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/inventory";
|
||||
const BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
struct Reader {
|
||||
access: String,
|
||||
secret: String,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
async fn create(dist: &DistCluster, target: &Value, action: &str) -> TestResult<Self> {
|
||||
let reader = Self {
|
||||
access: format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]),
|
||||
secret: Uuid::new_v4().simple().to_string(),
|
||||
};
|
||||
let policy_name = format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]);
|
||||
let bucket = target["bucket"].as_str().ok_or("target bucket")?;
|
||||
let prefix = target["attempt_prefix"]
|
||||
.as_str()
|
||||
.ok_or("target prefix")?
|
||||
.trim_end_matches('/');
|
||||
let resource = if action == "s3:ListBucket" {
|
||||
format!("arn:aws:s3:::{bucket}")
|
||||
} else {
|
||||
format!("arn:aws:s3:::{bucket}/{prefix}")
|
||||
};
|
||||
let policy =
|
||||
json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":action,"Resource":resource}]}).to_string();
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, secret, &reader.access, &reader.secret).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &reader.access).await?;
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
async fn request(&self, dist: &DistCluster, target: &Value, invocation: Uuid) -> TestResult<reqwest::Response> {
|
||||
timeout(
|
||||
BUDGET,
|
||||
signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(json!({"version":1,"invocation_id":invocation,"target":target}).to_string()),
|
||||
Some("application/json"),
|
||||
&self.access,
|
||||
&self.secret,
|
||||
),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Frame {
|
||||
version: u32,
|
||||
invocation_id: Uuid,
|
||||
target: Value,
|
||||
sequence: u64,
|
||||
payload: Payload,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
|
||||
enum Payload {
|
||||
Observation {
|
||||
disk_id: Uuid,
|
||||
key: String,
|
||||
version_id: Uuid,
|
||||
delete_marker: bool,
|
||||
},
|
||||
Complete {
|
||||
members: usize,
|
||||
observations: u64,
|
||||
},
|
||||
}
|
||||
|
||||
async fn read_frames(mut response: reqwest::Response) -> TestResult<(Vec<Frame>, bool)> {
|
||||
let mut pending = Vec::new();
|
||||
let mut frames = Vec::new();
|
||||
let mut clean = true;
|
||||
loop {
|
||||
match timeout(BUDGET, response.chunk()).await? {
|
||||
Ok(Some(chunk)) => {
|
||||
pending.extend_from_slice(&chunk);
|
||||
while let Some(end) = pending.iter().position(|byte| *byte == b'\n') {
|
||||
assert!(end < 1024 * 1024, "HTTP inventory line exceeds encoded bound");
|
||||
let line: Vec<_> = pending.drain(..=end).collect();
|
||||
frames.push(serde_json::from_slice(&line[..end])?);
|
||||
}
|
||||
assert!(pending.len() < 1024 * 1024, "unbounded incomplete HTTP line");
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(_) => {
|
||||
clean = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((frames, clean && pending.is_empty()))
|
||||
}
|
||||
|
||||
async fn seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value) -> TestResult {
|
||||
let (status, body) = user
|
||||
.request(
|
||||
dist,
|
||||
0,
|
||||
"/rustfs/admin/v3/plugin-artifacts/seal-writes",
|
||||
json!({"version":1,"target":target}).to_string(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::OK, "actual all-member closure: {body}");
|
||||
assert_eq!(serde_json::from_str::<Value>(&body)?["all_writes_sealed"], true);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn exact_inventory(frames: &[Frame], target: &Value, invocation: Uuid, versions: &BTreeSet<(String, Uuid, bool)>) {
|
||||
let mut observed = BTreeSet::new();
|
||||
let mut disks = BTreeSet::new();
|
||||
let mut count = 0u64;
|
||||
for (sequence, frame) in frames.iter().enumerate() {
|
||||
assert_eq!(frame.version, 1);
|
||||
assert_eq!(frame.invocation_id, invocation);
|
||||
assert_eq!(&frame.target, target);
|
||||
assert_eq!(frame.sequence, u64::try_from(sequence).expect("bounded fixture sequence"));
|
||||
match &frame.payload {
|
||||
Payload::Observation {
|
||||
disk_id,
|
||||
key,
|
||||
version_id,
|
||||
delete_marker,
|
||||
} => {
|
||||
assert!(!disk_id.is_nil() && !version_id.is_nil());
|
||||
assert!(
|
||||
observed.insert((*disk_id, key.clone(), *version_id, *delete_marker)),
|
||||
"duplicate physical observation"
|
||||
);
|
||||
disks.insert(*disk_id);
|
||||
count += 1;
|
||||
}
|
||||
Payload::Complete { members, observations } => {
|
||||
assert_eq!(sequence + 1, frames.len(), "Complete must be the last HTTP frame");
|
||||
assert_eq!(*members, 4);
|
||||
assert_eq!(*observations, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
|
||||
"A03I3A_HTTP_COMPLETE"
|
||||
);
|
||||
if !versions.is_empty() {
|
||||
assert_eq!(disks.len(), 4);
|
||||
}
|
||||
for disk in disks {
|
||||
let actual: BTreeSet<_> = observed
|
||||
.iter()
|
||||
.filter(|(id, ..)| *id == disk)
|
||||
.map(|(_, key, version, marker)| (key.clone(), *version, *marker))
|
||||
.collect();
|
||||
assert_eq!(&actual, versions, "A03I3A_EXACT_VERSIONS");
|
||||
}
|
||||
assert_eq!(observed.len(), versions.len() * 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_exact_versions() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
|
||||
let mut versions = BTreeSet::new();
|
||||
for (name, body) in [
|
||||
("same", b"first".to_vec()),
|
||||
("same", b"second".to_vec()),
|
||||
("zero", Vec::new()),
|
||||
] {
|
||||
let key = format!("{prefix}{name}");
|
||||
let output = timeout(BUDGET, dist.client(1)?.put_object().bucket(bucket).key(&key).body(body.into()).send()).await??;
|
||||
let version = Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?;
|
||||
versions.insert((key, version, false));
|
||||
}
|
||||
seal(&dist, &admin, &target).await?;
|
||||
// A wildcard action is still restricted to this exact attempt resource.
|
||||
// The separate authority case exercises the dedicated action alone.
|
||||
let reader = Reader::create(&dist, &target, "admin:*").await?;
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = reader.request(&dist, &target, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK, "A03I3A_HTTP_ROUTE");
|
||||
assert_eq!(
|
||||
response.headers().get("content-type").and_then(|value| value.to_str().ok()),
|
||||
Some("application/x-ndjson")
|
||||
);
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean, "Complete requires clean HTTP EOF");
|
||||
assert!(frames.len() > 2, "real multi-frame HTTP body");
|
||||
exact_inventory(&frames, &target, invocation, &versions);
|
||||
assert!(
|
||||
before == physical_snapshot(&dist, bucket)?,
|
||||
"inventory mutated physical artifact namespace"
|
||||
);
|
||||
let (empty, empty_admin) = create_artifact_attempt(&dist).await?;
|
||||
seal(&dist, &empty_admin, &empty).await?;
|
||||
let empty_reader = Reader::create(&dist, &empty, "admin:*").await?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = empty_reader.request(&dist, &empty, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean);
|
||||
exact_inventory(&frames, &empty, invocation, &BTreeSet::new());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_scoped_authority() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
seal(&dist, &admin, &target).await?;
|
||||
let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = reader.request(&dist, &target, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK, "dedicated inventory authority");
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean);
|
||||
exact_inventory(&frames, &target, invocation, &BTreeSet::new());
|
||||
let mut other_prefix = target.clone();
|
||||
other_prefix["attempt_prefix"] = json!(format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4()));
|
||||
let mut other_bucket = target.clone();
|
||||
other_bucket["bucket"] = json!("another-artifact-bucket");
|
||||
for denied in [&other_prefix, &other_bucket] {
|
||||
assert_eq!(
|
||||
reader.request(&dist, denied, Uuid::new_v4()).await?.status(),
|
||||
StatusCode::FORBIDDEN,
|
||||
"A03I3A_SCOPED_AUTHORITY"
|
||||
);
|
||||
}
|
||||
let (status, _) = admin
|
||||
.request(
|
||||
&dist,
|
||||
0,
|
||||
ROUTE,
|
||||
json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "A03I3A_SCOPED_AUTHORITY seal-only user");
|
||||
let listed = Reader::create(&dist, &target, "s3:ListBucket").await?;
|
||||
dist.client_with_credentials(0, &listed.access, &listed.secret)?
|
||||
.list_objects_v2()
|
||||
.bucket(target["bucket"].as_str().ok_or("bucket")?)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
listed.request(&dist, &target, Uuid::new_v4()).await?.status(),
|
||||
StatusCode::FORBIDDEN,
|
||||
"A03I3A_SCOPED_AUTHORITY S3 list-only user"
|
||||
);
|
||||
let root_response = signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string()),
|
||||
Some("application/json"),
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(root_response.status(), StatusCode::FORBIDDEN, "original scoped user required");
|
||||
for body in [
|
||||
json!({"version":1,"invocation_id":Uuid::nil(),"target":target}),
|
||||
json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target,"extra":true}),
|
||||
] {
|
||||
let response = signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(body.to_string()),
|
||||
Some("application/json"),
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "strict inventory request");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_incomplete(response: reqwest::Response) -> TestResult<Vec<Frame>> {
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::OK,
|
||||
"valid request must exercise the streaming failure boundary"
|
||||
);
|
||||
let (frames, _) = read_frames(response).await?;
|
||||
assert!(
|
||||
!frames.iter().any(|frame| matches!(frame.payload, Payload::Complete { .. })),
|
||||
"A03I3A_NO_FALSE_COMPLETE"
|
||||
);
|
||||
Ok(frames)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_incomplete_has_no_terminal() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
|
||||
// Real PUTs produce more data than a stalled HTTP/2 flow-control window.
|
||||
futures::stream::iter(0..256)
|
||||
.map(|index| {
|
||||
let client = dist.client(1);
|
||||
let key = format!("{prefix}slow/{index:04}/{}/{}", "a".repeat(190), "b".repeat(190));
|
||||
async move {
|
||||
let output = client?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(vec![1].into())
|
||||
.send()
|
||||
.await?;
|
||||
assert!(!Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?.is_nil());
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
|
||||
}
|
||||
})
|
||||
.buffer_unordered(8)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
seal(&dist, &admin, &target).await?;
|
||||
let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
|
||||
dist.cluster.stop_node(3)?;
|
||||
assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let mut wrong = target.clone();
|
||||
wrong["bucket_incarnation"] = json!(Uuid::new_v4());
|
||||
assert_incomplete(reader.request(&dist, &wrong, Uuid::new_v4()).await?).await?;
|
||||
// The configured endpoint map is ordered. Damage its final receiver, so
|
||||
// earlier real observations are emitted before the physical scan fails.
|
||||
let node = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.max_by_key(|node| format!("{}{}", node.url, node.data_dirs[0]))
|
||||
.ok_or("configured members")?;
|
||||
let metadata = std::path::Path::new(&node.data_dirs[0])
|
||||
.join(bucket)
|
||||
.join(format!("{prefix}slow/0000/{}/{}", "a".repeat(190), "b".repeat(190)))
|
||||
.join("xl.meta");
|
||||
let original = std::fs::read(&metadata)?;
|
||||
std::fs::write(&metadata, b"corrupt inventory metadata")?;
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let outcome = assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await;
|
||||
let after = physical_snapshot(&dist, bucket)?;
|
||||
std::fs::write(&metadata, original)?;
|
||||
let frames = outcome?;
|
||||
assert!(before == after, "failed inventory repaired or mutated a physical object");
|
||||
assert!(
|
||||
frames
|
||||
.iter()
|
||||
.any(|frame| matches!(frame.payload, Payload::Observation { .. })),
|
||||
"late physical failure must follow real HTTP observations"
|
||||
);
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.http2_prior_knowledge()
|
||||
.http2_initial_stream_window_size(1)
|
||||
.http2_initial_connection_window_size(1)
|
||||
.http2_adaptive_window(false)
|
||||
.build()?;
|
||||
let url = format!("{}{ROUTE}", dist.cluster.nodes[0].url);
|
||||
let request = |invocation| {
|
||||
json!({"version":1,"invocation_id":invocation,"target":target})
|
||||
.to_string()
|
||||
.into_bytes()
|
||||
};
|
||||
let response = crate::common::signed_request_with_client(
|
||||
&client,
|
||||
Method::POST,
|
||||
&url,
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
Some(request(Uuid::new_v4())),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
// Do not poll the body: its one-byte HTTP/2 receive window stalls transport.
|
||||
// This waits beyond the fixed30s service budget, not for readiness.
|
||||
tokio::time::sleep(Duration::from_secs(35)).await;
|
||||
assert_incomplete(response).await?;
|
||||
let response = crate::common::signed_request_with_client(
|
||||
&client,
|
||||
Method::POST,
|
||||
&url,
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
Some(request(Uuid::new_v4())),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
drop(response);
|
||||
let response = timeout(Duration::from_secs(10), reader.request(&dist, &target, Uuid::new_v4())).await??;
|
||||
let (frames, clean) = timeout(Duration::from_secs(10), read_frames(response)).await??;
|
||||
assert!(
|
||||
clean && matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
|
||||
"fresh scan must succeed promptly after an actual disconnected HTTP body"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_repair_tail_test::PhysicalBarrier;
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
|
||||
use crate::chaos::census_object_version_on_disk;
|
||||
use crate::common::init_logging;
|
||||
use crate::storage_api::artifact_rpc::{
|
||||
DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_protos::proto_gen::node_service::{RenameDataRequest, node_service_client::NodeServiceClient};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PUT_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
|
||||
|
||||
fn signed_put_request(body: RenameDataRequest, audience: &str) -> TestResult<Request<RenameDataRequest>> {
|
||||
let canonical = rustfs_protos::canonical_artifact_rename_data_request_body(&body)?;
|
||||
let mut request = Request::new(body);
|
||||
request.set_timeout(Duration::from_secs(15));
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.ok_or("missing canonical digest")?
|
||||
.to_str()?
|
||||
.to_owned();
|
||||
request.metadata_mut().as_mut().extend(gen_tonic_signature_headers(
|
||||
audience,
|
||||
TONIC_RPC_PREFIX.trim_start_matches('/'),
|
||||
"ArtifactRenameData",
|
||||
Some(&digest),
|
||||
)?);
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
fn publication_snapshot(paths: &[&Path]) -> TestResult<BTreeMap<PathBuf, Option<Vec<u8>>>> {
|
||||
let mut snapshot = BTreeMap::new();
|
||||
for root in paths {
|
||||
for entry in walkdir::WalkDir::new(root) {
|
||||
let entry = entry?;
|
||||
let contents = if entry.file_type().is_dir() {
|
||||
None
|
||||
} else {
|
||||
Some(std::fs::read(entry.path())?)
|
||||
};
|
||||
snapshot.insert(entry.path().to_path_buf(), contents);
|
||||
}
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn assert_raw_put_closed(body: &RenameDataRequest, url: &str, audience: &str, paths: &[&Path]) -> TestResult {
|
||||
let before = publication_snapshot(paths)?;
|
||||
// Re-sign the exact original request; replay protection is not the rejection.
|
||||
let request = signed_put_request(body.clone(), audience)?;
|
||||
let response = timeout(Duration::from_secs(20), async {
|
||||
let mut client = NodeServiceClient::connect(url.to_owned()).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?.into_inner())
|
||||
})
|
||||
.await??;
|
||||
assert!(!response.success, "sealed attempt admitted another ordinary PUT");
|
||||
let error = response.error.ok_or("rejected raw PUT omitted error")?;
|
||||
assert_eq!(error.code, DiskError::Io(std::io::ErrorKind::PermissionDenied.into()).to_u32());
|
||||
assert_eq!(error.error_info, "io error artifact attempt mutation domain is closed");
|
||||
assert_eq!(publication_snapshot(paths)?, before, "rejected raw PUT mutated staged or published files");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_put_seal_waits_for_disconnected_physical_tail() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
|
||||
let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let key = format!("{prefix}detached-put.bin");
|
||||
let payload = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(payload.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?;
|
||||
let request_body = json!({"version": 1, "target": target}).to_string();
|
||||
let baselines = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, &key, Some(version)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
assert!(
|
||||
baselines
|
||||
.iter()
|
||||
.all(|baseline| baseline.is_complete() && !baseline.expected_part_numbers.is_empty())
|
||||
);
|
||||
|
||||
let node = &dist.cluster.nodes[3];
|
||||
let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
|
||||
let object = disk.join(bucket).join(&key);
|
||||
let metadata_bytes = std::fs::read(object.join("xl.meta"))?;
|
||||
let fi = FileMeta::load(&metadata_bytes)?.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
let admitted_file_info = serde_json::to_value(&fi)?;
|
||||
assert!(!fi.is_healing(), "ordinary publication must not carry the repair marker");
|
||||
let data_dir = fi
|
||||
.data_dir
|
||||
.ok_or("physical-tail fixture requires external shards")?
|
||||
.to_string();
|
||||
let source = Uuid::new_v4().to_string();
|
||||
let staging = disk.join(".rustfs.sys/tmp").join(&source);
|
||||
std::fs::create_dir_all(&staging)?;
|
||||
let destination = object.join(&data_dir);
|
||||
// Replay the receiver publication of a real ordinary PUT while its attempt
|
||||
// remains open. Preserve its referenced version metadata. The admitted
|
||||
// owned operation may still finish metadata publication after RPC detach.
|
||||
std::fs::rename(&destination, staging.join(&data_dir))?;
|
||||
let barrier = PhysicalBarrier::arm(&disk, &destination)?;
|
||||
let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
|
||||
let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
|
||||
let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
|
||||
let put = RenameDataRequest {
|
||||
disk: disk_id.to_string(),
|
||||
src_volume: ".rustfs.sys/tmp".to_owned(),
|
||||
src_path: source,
|
||||
dst_volume: bucket.to_owned(),
|
||||
dst_path: key.clone(),
|
||||
file_info: serde_json::to_string(&fi)?,
|
||||
file_info_bin: Default::default(),
|
||||
scanner_publication_lease_token: Default::default(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
};
|
||||
let request = signed_put_request(put.clone(), &node.address)?;
|
||||
// Direct RPC takes no coordinator attempt lock and owns its own connection.
|
||||
let url = node.url.clone();
|
||||
let mut rpc = tokio::spawn(async move {
|
||||
timeout(Duration::from_secs(30), async move {
|
||||
let mut client = NodeServiceClient::connect(url).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?)
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::select! {
|
||||
result = &mut rpc => {
|
||||
let response = result???.into_inner();
|
||||
panic!(
|
||||
"ordinary PUT receiver completed before the physical barrier: success={}, error={:?}",
|
||||
response.success, response.error
|
||||
);
|
||||
}
|
||||
arrived = barrier.observed("arrived") => arrived?,
|
||||
}
|
||||
assert!(!destination.exists(), "physical PUT escaped its barrier");
|
||||
rpc.abort();
|
||||
assert!(rpc.await.expect_err("disconnect the ordinary PUT waiter").is_cancelled());
|
||||
// The ordinary PUT wrapper retains an owned task. Observe its RPC-facing
|
||||
// waiter dropping, rather than cancellation of the continuing disk task.
|
||||
barrier.observed("rpc-waiter-dropped").await?;
|
||||
|
||||
let seal = user.request(&dist, 0, PUT_SEAL_ROUTE, request_body.clone());
|
||||
tokio::pin!(seal);
|
||||
let closed = wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async {
|
||||
Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-put-seals"))
|
||||
.min_depth(2)
|
||||
.max_depth(2)
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| entry.file_name() == "closed"))
|
||||
},
|
||||
"PUT receiver persisted closing before physical drain",
|
||||
);
|
||||
tokio::pin!(closed);
|
||||
tokio::select! {
|
||||
result = &mut seal => {
|
||||
let (status, response) = result?;
|
||||
panic!("PUT seal returned before its blocked receiver closed admission: {status} {response}");
|
||||
}
|
||||
result = &mut closed => result?,
|
||||
}
|
||||
let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"PUT seal confirmed an unfinished physical tail: {response}"
|
||||
);
|
||||
assert!(response.contains("artifact operation unconfirmed"));
|
||||
assert!(!destination.exists());
|
||||
assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
|
||||
assert!(!destination.exists(), "a second admission published while the worker was paused");
|
||||
|
||||
barrier.release()?;
|
||||
wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async { Ok(destination.join("part.1").is_file()) },
|
||||
"detached ordinary PUT worker finished",
|
||||
)
|
||||
.await?;
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
// Seal confirmation is the boundary: already admitted work may re-encode
|
||||
// metadata before this point, but the complete FileInfo must be unchanged.
|
||||
let published_objects: Vec<_> = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| PathBuf::from(&node.data_dirs[0]).join(bucket).join(&key))
|
||||
.collect();
|
||||
let published_paths: Vec<_> = published_objects.iter().map(PathBuf::as_path).collect();
|
||||
let sealed_snapshot = publication_snapshot(&published_paths)?;
|
||||
let sealed_file_info =
|
||||
FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
assert_eq!(
|
||||
serde_json::to_value(sealed_file_info)?,
|
||||
admitted_file_info,
|
||||
"admitted PUT changed FileInfo semantics"
|
||||
);
|
||||
for (index, baseline) in baselines.iter().enumerate() {
|
||||
assert!(
|
||||
census_object_version_on_disk(Path::new(&dist.cluster.nodes[index].data_dirs[0]), bucket, &key, Some(version))?
|
||||
.matches_manifest(baseline)
|
||||
);
|
||||
assert_exact_version(&dist.client(index)?, bucket, &key, version, &payload).await?;
|
||||
}
|
||||
|
||||
// Stage the exact request again. A source metadata sentinel makes a missing
|
||||
// admission check observable even if the existing destination later causes
|
||||
// ENOTEMPTY: ordinary rename rewrites source metadata before that syscall.
|
||||
let staged_data = staging.join(&data_dir);
|
||||
std::fs::create_dir_all(&staged_data)?;
|
||||
for entry in std::fs::read_dir(&destination)? {
|
||||
let entry = entry?;
|
||||
assert!(entry.file_type()?.is_file(), "unexpected nested shard fixture");
|
||||
std::fs::copy(entry.path(), staged_data.join(entry.file_name()))?;
|
||||
}
|
||||
std::fs::write(staging.join("xl.meta"), b"closed-admission-must-preserve-this-staged-metadata")?;
|
||||
assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(&key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 1, "raw PUT retries created another version");
|
||||
assert_eq!(listed.versions()[0].version_id(), Some(version));
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
assert!(
|
||||
publication_snapshot(&published_paths)? == sealed_snapshot,
|
||||
"published files changed after the first successful physical-drain confirmation"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
|
||||
use crate::chaos::census_object_version_on_disk;
|
||||
use crate::common::init_logging;
|
||||
use crate::storage_api::artifact_rpc::{TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
|
||||
use http::StatusCode;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
ArtifactRepairRenameDataRequest, RenameDataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use uuid::Uuid;
|
||||
|
||||
const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
|
||||
|
||||
pub(super) struct PhysicalBarrier(PathBuf);
|
||||
|
||||
impl PhysicalBarrier {
|
||||
pub(super) fn arm(disk: &Path, destination: &Path) -> TestResult<Self> {
|
||||
let path = disk.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
std::fs::write(&path, destination.to_str().ok_or("non-UTF8 fixture destination")?)?;
|
||||
Ok(Self(path))
|
||||
}
|
||||
|
||||
pub(super) async fn observed(&self, stage: &str) -> TestResult {
|
||||
wait_until(Duration::from_secs(15), || async { Ok(self.0.with_extension(stage).is_file()) }, stage).await
|
||||
}
|
||||
|
||||
pub(super) fn release(&self) -> TestResult {
|
||||
std::fs::write(self.0.with_extension("release"), b"release physical worker")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PhysicalBarrier {
|
||||
fn drop(&mut self) {
|
||||
// Release even on assertion failure; the worker also has its own deadline.
|
||||
let _ = self.release();
|
||||
let _ = std::fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_waits_for_disconnected_physical_tail() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
// Match the other raw internode signature tests sharing the OnceLock.
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
|
||||
let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let key = format!("{prefix}detached-rename.bin");
|
||||
let body = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?;
|
||||
let request_body = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
|
||||
let node = &dist.cluster.nodes[3];
|
||||
let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
|
||||
let object = disk.join(bucket).join(&key);
|
||||
let metadata = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
|
||||
let mut fi = metadata.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
fi.set_healing();
|
||||
let data_dir = fi
|
||||
.data_dir
|
||||
.ok_or("physical-tail fixture requires external shard files")?
|
||||
.to_string();
|
||||
let baseline = census_object_version_on_disk(&disk, bucket, &key, Some(version))?;
|
||||
assert!(baseline.is_complete() && !baseline.expected_part_numbers.is_empty());
|
||||
let source = Uuid::new_v4().to_string();
|
||||
let staging = disk.join(".rustfs.sys/tmp").join(&source);
|
||||
std::fs::create_dir_all(&staging)?;
|
||||
let destination = object.join(&data_dir);
|
||||
std::fs::rename(&destination, staging.join(&data_dir))?;
|
||||
let barrier = PhysicalBarrier::arm(&disk, &destination)?;
|
||||
let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
|
||||
let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
|
||||
let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
|
||||
let repair = ArtifactRepairRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: disk_id.to_string(),
|
||||
src_volume: ".rustfs.sys/tmp".to_owned(),
|
||||
src_path: source,
|
||||
dst_volume: bucket.to_owned(),
|
||||
dst_path: key.clone(),
|
||||
file_info: serde_json::to_string(&fi)?,
|
||||
file_info_bin: Default::default(),
|
||||
scanner_publication_lease_token: Default::default(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&repair)?;
|
||||
let mut request = Request::new(repair);
|
||||
request.set_timeout(Duration::from_secs(15));
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.ok_or("missing canonical digest")?
|
||||
.to_str()?
|
||||
.to_owned();
|
||||
let headers = gen_tonic_signature_headers(
|
||||
&node.address,
|
||||
TONIC_RPC_PREFIX.trim_start_matches('/'),
|
||||
"ArtifactRepairRenameData",
|
||||
Some(&digest),
|
||||
)?;
|
||||
request.metadata_mut().as_mut().extend(headers);
|
||||
// An independent channel and direct receiver RPC take no coordinator attempt
|
||||
// lock. Dropping this task closes the client stream while rename is paused.
|
||||
let url = node.url.clone();
|
||||
let mut rpc = tokio::spawn(async move {
|
||||
timeout(Duration::from_secs(30), async move {
|
||||
let mut client = NodeServiceClient::connect(url).await?;
|
||||
client
|
||||
.artifact_repair_rename_data(request)
|
||||
.await
|
||||
.map_err(|error| -> Box<dyn std::error::Error + Send + Sync> { Box::new(error) })
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::select! {
|
||||
result = &mut rpc => {
|
||||
let response = result???.into_inner();
|
||||
panic!(
|
||||
"repair receiver completed before the physical barrier: success={}, error={:?}",
|
||||
response.success, response.error
|
||||
);
|
||||
}
|
||||
arrived = barrier.observed("arrived") => arrived?,
|
||||
}
|
||||
assert!(!destination.exists(), "physical publication ran before its barrier");
|
||||
rpc.abort();
|
||||
assert!(rpc.await.expect_err("cancel the disconnected RPC waiter").is_cancelled());
|
||||
// This acknowledgement is emitted by the receiver's dropped async waiter,
|
||||
// not by the client and not by a delayed response.
|
||||
barrier.observed("waiter-dropped").await?;
|
||||
|
||||
let seal = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone());
|
||||
tokio::pin!(seal);
|
||||
let closed = wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async {
|
||||
Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-repair-seals"))
|
||||
.min_depth(2)
|
||||
.max_depth(2)
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| entry.file_name() == "closed"))
|
||||
},
|
||||
"receiver persisted repair closure while its physical worker remains paused",
|
||||
);
|
||||
tokio::pin!(closed);
|
||||
tokio::select! {
|
||||
result = &mut seal => {
|
||||
let (status, response) = result?;
|
||||
panic!("repair seal returned before the blocked receiver closed admission: {status} {response}");
|
||||
}
|
||||
result = &mut closed => result?,
|
||||
}
|
||||
// Cover a complete receiver drain timeout, so early physical-owner release
|
||||
// causes a successful response here and fails this runtime assertion.
|
||||
let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"seal confirmed an unfinished physical tail: {response}"
|
||||
);
|
||||
assert!(response.contains("artifact operation unconfirmed"));
|
||||
assert!(!destination.exists(), "worker escaped the physical barrier");
|
||||
barrier.release()?;
|
||||
wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async { Ok(destination.join("part.1").is_file()) },
|
||||
"detached physical rename completes after release",
|
||||
)
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(45),
|
||||
|| async {
|
||||
let (status, response) = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone()).await?;
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed") {
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "repair seal did not confirm after physical drain: {response}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&response)?,
|
||||
json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"exact retry confirms all four receivers after physical drain",
|
||||
)
|
||||
.await?;
|
||||
assert!(census_object_version_on_disk(&disk, bucket, &key, Some(version))?.matches_manifest(&baseline));
|
||||
assert_exact_version(&dist.client(0)?, bucket, &key, version, &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, payload_for, unique_bucket, wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post, start_root_heal_when_control_ready};
|
||||
use crate::common::init_logging;
|
||||
use http::StatusCode;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
|
||||
const HEAL_BODY: &str =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
const HEAL_BUDGET: Duration = Duration::from_secs(120);
|
||||
|
||||
pub(super) struct RepairObject {
|
||||
pub(super) key: String,
|
||||
pub(super) version: String,
|
||||
pub(super) body: Vec<u8>,
|
||||
manifests: Vec<VersionShardCensus>,
|
||||
}
|
||||
|
||||
fn roots(dist: &DistCluster) -> Vec<PathBuf> {
|
||||
dist.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| PathBuf::from(&node.data_dirs[0]))
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl RepairObject {
|
||||
pub(super) async fn put(dist: &DistCluster, bucket: &str, key: String) -> TestResult<Self> {
|
||||
let body = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?.to_owned();
|
||||
let manifests = roots(dist)
|
||||
.iter()
|
||||
.map(|disk| census_object_version_on_disk(disk, bucket, &key, Some(&version)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
for manifest in &manifests {
|
||||
assert!(manifest.is_complete(), "repair needs a complete baseline: {manifest:?}");
|
||||
assert!(!manifest.expected_part_numbers.is_empty(), "repair must use actual shard files");
|
||||
}
|
||||
Ok(Self {
|
||||
key,
|
||||
version,
|
||||
body,
|
||||
manifests,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn part(&self, disk: &Path, bucket: &str, node: usize) -> TestResult<PathBuf> {
|
||||
let data_dir = self.manifests[node]
|
||||
.data_dir
|
||||
.as_ref()
|
||||
.ok_or("external shard omitted data directory")?;
|
||||
Ok(disk.join(bucket).join(&self.key).join(data_dir).join("part.1"))
|
||||
}
|
||||
|
||||
pub(super) async fn assert_repaired(&self, dist: &DistCluster, bucket: &str) -> TestResult {
|
||||
wait_until(
|
||||
HEAL_BUDGET,
|
||||
|| async {
|
||||
for (disk, baseline) in roots(dist).iter().zip(&self.manifests) {
|
||||
if !census_object_version_on_disk(disk, bucket, &self.key, Some(&self.version))?.matches_manifest(baseline) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"every configured receiver restored the exact version and original shard fingerprints",
|
||||
)
|
||||
.await?;
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_exact_version(&dist.client(node)?, bucket, &self.key, &self.version, &self.body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn start_heal(dist: &DistCluster, bucket: &str, prefix: &str, remove: bool) -> TestResult<String> {
|
||||
let url = format!("{}/rustfs/admin/v3/heal/{bucket}/{prefix}?forceStart=true", dist.cluster.nodes[0].url);
|
||||
let body = if remove {
|
||||
HEAL_BODY.replace("\"remove\":false", "\"remove\":true")
|
||||
} else {
|
||||
HEAL_BODY.to_owned()
|
||||
};
|
||||
let response = timeout(
|
||||
HEAL_BUDGET,
|
||||
start_root_heal_when_control_ready(&url, &body, &dist.cluster.access_key, &dist.cluster.secret_key),
|
||||
)
|
||||
.await??;
|
||||
let response: Value = serde_json::from_str(&response)?;
|
||||
Ok(response["clientToken"]
|
||||
.as_str()
|
||||
.ok_or("heal omitted its task token")?
|
||||
.to_owned())
|
||||
}
|
||||
|
||||
pub(super) async fn finish_heal(dist: &DistCluster, token: &str, rejected_key: Option<&str>) -> TestResult {
|
||||
let url = format!("{}/rustfs/admin/v3/heal/?clientToken={token}", dist.cluster.nodes[0].url);
|
||||
wait_until(
|
||||
HEAL_BUDGET,
|
||||
|| async {
|
||||
let response = signed_admin_post(&url, None, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
let status: Value = serde_json::from_str(&response)?;
|
||||
if status["summary"] == "running" {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Some(key) = rejected_key {
|
||||
assert_eq!(status["summary"], "stopped", "closed repair must report failure: {status}");
|
||||
assert!(
|
||||
status["outcome"]["counters"]["failed"]
|
||||
.as_u64()
|
||||
.is_some_and(|count| count > 0)
|
||||
);
|
||||
assert!(
|
||||
status["outcome"]["objects"].as_array().is_some_and(|objects| objects
|
||||
.iter()
|
||||
.any(|object| { object["identity"]["object"] == key && object["disposition"]["state"] == "failed" })),
|
||||
"heal must actually attempt the rejected object: {status}"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(status["summary"], "finished", "heal did not complete: {status}");
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"exact admin heal task reached its expected terminal outcome",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn repair_seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
|
||||
wait_until(
|
||||
Duration::from_secs(45),
|
||||
|| async {
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
let (status, body) = user.request(dist, 0, REPAIR_SEAL_ROUTE, request).await?;
|
||||
if !confirmed {
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver repair seal: {body}");
|
||||
assert!(body.contains("<Code>ServiceUnavailable</Code>"));
|
||||
assert!(body.contains("artifact operation unconfirmed"));
|
||||
return Ok(true);
|
||||
}
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE
|
||||
&& body.contains("<Code>ServiceUnavailable</Code>")
|
||||
&& body.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "repair seal failed: {body}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&body)?,
|
||||
json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"repair seal confirms every configured receiver",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_shard_and_minority_metadata_repair_after_put_seal() -> TestResult {
|
||||
init_logging();
|
||||
// Sub-quorum metadata recovery supports genuine legacy shards only.
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
|
||||
let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
|
||||
let later = dist
|
||||
.client(2)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&shard.key)
|
||||
.body(b"newer-version".to_vec().into())
|
||||
.send()
|
||||
.await?;
|
||||
let later_version = later.version_id().ok_or("second PUT omitted version ID")?.to_owned();
|
||||
assert_ne!(shard.version, later_version);
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
|
||||
dist.cluster.stop();
|
||||
let disks = roots(&dist);
|
||||
std::fs::remove_file(shard.part(&disks[3], bucket, 3)?)?;
|
||||
for disk in &disks[1..] {
|
||||
std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
|
||||
}
|
||||
assert!(!census_object_version_on_disk(&disks[3], bucket, &shard.key, Some(&shard.version))?.is_complete());
|
||||
assert!(disks[0].join(bucket).join(&metadata.key).join("xl.meta").is_file());
|
||||
assert!(
|
||||
disks[1..]
|
||||
.iter()
|
||||
.all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
|
||||
);
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let token = start_heal(&dist, bucket, prefix, false).await?;
|
||||
shard.assert_repaired(&dist, bucket).await?;
|
||||
metadata.assert_repaired(&dist, bucket).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
assert_exact_version(&dist.client(3)?, bucket, &shard.key, &later_version, b"newer-version").await?;
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 3, "repair must not publish replacement versions");
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_denies_shard_and_metadata_repair_after_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
|
||||
let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
repair_seal(&dist, &user, &target, true).await?;
|
||||
let ordinary_bucket = unique_bucket("repair-control");
|
||||
dist.create_bucket(&ordinary_bucket).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary_bucket).await?;
|
||||
let ordinary = RepairObject::put(&dist, &ordinary_bucket, "ordinary.bin".to_owned()).await?;
|
||||
let disks = roots(&dist);
|
||||
let part = shard.part(&disks[3], bucket, 3)?;
|
||||
std::fs::remove_file(&part)?;
|
||||
for disk in &disks[1..] {
|
||||
std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
|
||||
}
|
||||
for restarted in [false, true] {
|
||||
if restarted {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
// The same receivers must still execute an ordinary bucket repair.
|
||||
std::fs::remove_file(ordinary.part(&disks[3], &ordinary_bucket, 3)?)?;
|
||||
let control = start_heal(&dist, &ordinary_bucket, &ordinary.key, false).await?;
|
||||
ordinary.assert_repaired(&dist, &ordinary_bucket).await?;
|
||||
finish_heal(&dist, &control, None).await?;
|
||||
for object in [&shard, &metadata] {
|
||||
let token = start_heal(&dist, bucket, &object.key, false).await?;
|
||||
finish_heal(&dist, &token, Some(&object.key)).await?;
|
||||
}
|
||||
assert!(!part.exists(), "sealed repair recreated the missing shard");
|
||||
assert!(
|
||||
disks[1..]
|
||||
.iter()
|
||||
.all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
|
||||
);
|
||||
assert_exact_version(&dist.client(2)?, bucket, &shard.key, &shard.version, &shard.body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_requires_every_receiver_on_exact_retry() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
// Repeat after global confirmation: its persisted receipt cannot replace a
|
||||
// fresh acknowledgement from an offline configured receiver.
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
repair_seal(&dist, &user, &target, false).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
repair_seal(&dist, &user, &target, true).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_dangling_repair_preserves_surviving_metadata() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let object = RepairObject::put(&dist, bucket, format!("{prefix}dangling.bin")).await?;
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
dist.cluster.stop();
|
||||
let disks = roots(&dist);
|
||||
let survivor = disks[0].join(bucket).join(&object.key).join("xl.meta");
|
||||
let before = std::fs::read(&survivor)?;
|
||||
for (node, disk) in disks.iter().enumerate() {
|
||||
std::fs::remove_file(object.part(disk, bucket, node)?)?;
|
||||
if node != 0 {
|
||||
std::fs::remove_file(disk.join(bucket).join(&object.key).join("xl.meta"))?;
|
||||
}
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let token = start_heal(&dist, bucket, &object.key, true).await?;
|
||||
finish_heal(&dist, &token, Some(&object.key)).await?;
|
||||
assert_eq!(
|
||||
std::fs::read(survivor)?,
|
||||
before,
|
||||
"distributed repair must not authorize managed dangling deletion"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, put_object, s3_probe_client, unique_bucket, wait_for_ready,
|
||||
wait_until,
|
||||
};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, admin_request, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use http::{Method, StatusCode};
|
||||
use serde_json::{Value, json};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
const BUCKET_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets";
|
||||
const INSPECT_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets/inspect";
|
||||
const SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
|
||||
const ADMIN_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
pub(super) struct ArtifactAdmin {
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
}
|
||||
|
||||
impl ArtifactAdmin {
|
||||
pub(super) async fn request(
|
||||
&self,
|
||||
dist: &DistCluster,
|
||||
node: usize,
|
||||
route: &str,
|
||||
body: String,
|
||||
) -> TestResult<(StatusCode, String)> {
|
||||
timeout(
|
||||
ADMIN_BUDGET,
|
||||
admin_request(
|
||||
&dist.cluster.nodes[node].url,
|
||||
Method::POST,
|
||||
route,
|
||||
Some(body),
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_artifact_admin(dist: &DistCluster, bucket: &str, prefix: &str) -> TestResult<ArtifactAdmin> {
|
||||
let user = ArtifactAdmin {
|
||||
access_key: format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]),
|
||||
secret_key: Uuid::new_v4().simple().to_string(),
|
||||
};
|
||||
let policy_name = format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]);
|
||||
let policy = json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["admin:CreatePluginArtifactBucket", "admin:GetPluginArtifactBucket"],
|
||||
"Resource": format!("arn:aws:s3:::{bucket}"),
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["admin:SealPluginArtifactAttempt", "admin:SealPluginArtifactRepairs", "admin:SealPluginArtifactWrites"],
|
||||
"Resource": format!("arn:aws:s3:::{bucket}/{}", prefix.trim_end_matches('/')),
|
||||
},
|
||||
],
|
||||
})
|
||||
.to_string();
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, secret, &user.access_key, &user.secret_key).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &user.access_key).await?;
|
||||
Ok(user)
|
||||
}
|
||||
|
||||
pub(super) async fn create_artifact_attempt(dist: &DistCluster) -> TestResult<(Value, ArtifactAdmin)> {
|
||||
let bucket = unique_bucket("artifact");
|
||||
let prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
let user = create_artifact_admin(dist, &bucket, &prefix).await?;
|
||||
let body = json!({"version": 1, "bucket": bucket, "base_prefix": "outputs/"}).to_string();
|
||||
let (status, response) = user.request(dist, 0, BUCKET_ROUTE, body).await?;
|
||||
assert_eq!(status, StatusCode::OK, "artifact bucket creation failed: {response}");
|
||||
let receipt: Value = serde_json::from_str(&response)?;
|
||||
assert_eq!(receipt["version"], 1);
|
||||
assert_eq!(receipt["bucket"], bucket);
|
||||
assert_eq!(receipt["base_prefix"], "outputs/");
|
||||
assert_eq!(receipt["versioning_enabled"], true);
|
||||
let incarnation = receipt["bucket_incarnation"]
|
||||
.as_str()
|
||||
.ok_or("artifact receipt omitted incarnation")?;
|
||||
assert!(!Uuid::parse_str(incarnation)?.is_nil());
|
||||
let target = json!({
|
||||
"bucket": bucket,
|
||||
"bucket_incarnation": incarnation,
|
||||
"attempt_prefix": prefix,
|
||||
});
|
||||
assert_bucket_identity(dist, &user, 1, &target).await?;
|
||||
Ok((target, user))
|
||||
}
|
||||
|
||||
async fn assert_bucket_identity(dist: &DistCluster, user: &ArtifactAdmin, node: usize, target: &Value) -> TestResult {
|
||||
let body = json!({"version": 1, "bucket": target["bucket"], "base_prefix": "outputs/"}).to_string();
|
||||
wait_until(
|
||||
ADMIN_BUDGET,
|
||||
|| async {
|
||||
let (status, response) = user.request(dist, node, INSPECT_ROUTE, body.clone()).await?;
|
||||
// IAM peers may not have received the new user and policy yet.
|
||||
// Storage failures must fail immediately, including the old gate's 503.
|
||||
if matches!(status, StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED)
|
||||
&& (response.contains("<Code>AccessDenied</Code>") || response.contains("<Code>InvalidAccessKeyId</Code>"))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "cross-node artifact inspect failed: {response}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&response)?,
|
||||
json!({
|
||||
"version": 1,
|
||||
"bucket": target["bucket"],
|
||||
"base_prefix": "outputs/",
|
||||
"bucket_incarnation": target["bucket_incarnation"],
|
||||
"versioning_enabled": true,
|
||||
}),
|
||||
"inspect must return the created managed bucket identity"
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"artifact IAM user and policy propagation",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn confirm_seal(dist: &DistCluster, user: &ArtifactAdmin, request: &str, target: &Value) -> TestResult {
|
||||
wait_until(
|
||||
ADMIN_BUDGET,
|
||||
|| async {
|
||||
let (status, response) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
|
||||
// HTTP readiness does not confirm that remote disk handles have
|
||||
// recovered. Retry only the API's explicit same-identity response.
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE
|
||||
&& response.contains("<Code>ServiceUnavailable</Code>")
|
||||
&& response.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "artifact seal failed: {response}");
|
||||
let receipt: Value = serde_json::from_str(&response)?;
|
||||
assert_eq!(receipt, json!({"version": 1, "target": target, "ordinary_put_sealed": true}));
|
||||
Ok(true)
|
||||
},
|
||||
"exact artifact seal confirmation after receiver recovery",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn assert_seal_unconfirmed(dist: &DistCluster, user: &ArtifactAdmin, request: &str) -> TestResult {
|
||||
let (status, body) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver seal response: {body}");
|
||||
assert!(
|
||||
body.contains("<Code>ServiceUnavailable</Code>"),
|
||||
"seal failure lost its service code: {body}"
|
||||
);
|
||||
assert!(
|
||||
body.contains("artifact operation unconfirmed"),
|
||||
"seal failure lost its retry contract: {body}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn assert_exact_version(client: &Client, bucket: &str, key: &str, version: &str, body: &[u8]) -> TestResult {
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.get_object().bucket(bucket).key(key).version_id(version).send(),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(output.version_id(), Some(version));
|
||||
let actual = timeout(Duration::from_secs(30), output.body.collect()).await??.into_bytes();
|
||||
assert_eq!(actual.as_ref(), body);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_put_denied(client: &Client, ordinary_bucket: &str, bucket: &str, key: &str) -> TestResult {
|
||||
// The same receiver must still accept a normal write before its sealed probe.
|
||||
put_object(client, ordinary_bucket, key, b"ordinary-control".to_vec()).await?;
|
||||
assert_object_bytes(client, ordinary_bucket, key, b"ordinary-control").await?;
|
||||
let client = s3_probe_client(client);
|
||||
let result = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(b"must-not-publish".to_vec().into())
|
||||
.send(),
|
||||
)
|
||||
.await?;
|
||||
let error = result.expect_err("sealed artifact attempt accepted a new PUT");
|
||||
let service = error
|
||||
.as_service_error()
|
||||
.ok_or("sealed PUT failed without a service rejection")?;
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(500),
|
||||
"sealed PUT must return its current HTTP status"
|
||||
);
|
||||
assert_eq!(service.code(), Some("InternalError"), "sealed PUT must return its current service code");
|
||||
// StorageError::Io is redacted at the API boundary; the sealed reason is
|
||||
// not exposed. Success controls and exact inventory checks prove the effect.
|
||||
assert_eq!(
|
||||
service.message(),
|
||||
Some("We encountered an internal error, please try again."),
|
||||
"sealed PUT must retain the public redacted message"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_put_seal_survives_full_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
|
||||
let key = format!("{prefix}result.bin");
|
||||
let mut versions = Vec::new();
|
||||
for (node, body) in [(1, b"first-version".as_slice()), (2, b"second-version".as_slice())] {
|
||||
let client = dist.client(node)?;
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.put_object().bucket(bucket).key(&key).body(body.to_vec().into()).send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
|
||||
assert!(!Uuid::parse_str(&version)?.is_nil());
|
||||
assert_exact_version(&dist.client(3)?, bucket, &key, &version, body).await?;
|
||||
versions.push((version, body.to_vec()));
|
||||
}
|
||||
assert_ne!(versions[0].0, versions[1].0);
|
||||
let ordinary = unique_bucket("ordinary");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
put_object(&dist.client(2)?, &ordinary, "normal.bin", b"before-seal".to_vec()).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &ordinary, "normal.bin", b"before-seal").await?;
|
||||
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
let other_prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
for phase in ["sealed", "restarted"] {
|
||||
if phase == "restarted" {
|
||||
dist.cluster.stop();
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
}
|
||||
assert_bucket_identity(&dist, &user, 2, &target).await?;
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
// A second attempt in the managed bucket must remain writable;
|
||||
// a broken bucket-wide write path cannot stand in for sealing.
|
||||
let client = dist.client(node)?;
|
||||
let other_key = format!("{other_prefix}{phase}-{node}.bin");
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&other_key)
|
||||
.body(b"other-attempt".to_vec().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("other attempt PUT omitted version ID")?;
|
||||
assert!(!Uuid::parse_str(version)?.is_nil());
|
||||
assert_exact_version(
|
||||
&dist.client((node + 1) % dist.cluster.nodes.len())?,
|
||||
bucket,
|
||||
&other_key,
|
||||
version,
|
||||
b"other-attempt",
|
||||
)
|
||||
.await?;
|
||||
assert_put_denied(&dist.client(node)?, &ordinary, bucket, &format!("{prefix}{phase}-{node}.bin")).await?;
|
||||
for (version, body) in &versions {
|
||||
assert_exact_version(&dist.client(node)?, bucket, &key, version, body).await?;
|
||||
}
|
||||
}
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 2, "denied PUT left an artifact version: {listed:?}");
|
||||
assert!(listed.delete_markers().is_empty(), "artifact seal created a delete marker");
|
||||
put_object(&dist.client(1)?, &ordinary, "normal.bin", phase.as_bytes().to_vec()).await?;
|
||||
assert_object_bytes(&dist.client(2)?, &ordinary, "normal.bin", phase.as_bytes()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_seal_requires_every_configured_receiver_on_exact_retry() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
|
||||
let key = format!("{prefix}existing.bin");
|
||||
let output = dist
|
||||
.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(b"existing".to_vec().into())
|
||||
.send()
|
||||
.await?;
|
||||
let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
|
||||
assert_exact_version(&dist.client(3)?, bucket, &key, &version, b"existing").await?;
|
||||
let ordinary = unique_bucket("ordinary");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
|
||||
// The second iteration starts from a fully acknowledged global seal. Its
|
||||
// persisted record cannot replace a fresh acknowledgement from every disk.
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
assert_seal_unconfirmed(&dist, &user, &request).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
}
|
||||
assert_put_denied(&dist.client(3)?, &ordinary, bucket, &format!("{prefix}late.bin")).await?;
|
||||
assert_exact_version(&dist.client(2)?, bucket, &key, &version, b"existing").await?;
|
||||
let listed = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(0)?.list_object_versions().bucket(bucket).prefix(prefix).send(),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(listed.versions().len(), 1, "rejected PUT added a version");
|
||||
assert!(listed.delete_markers().is_empty(), "seal added a delete marker");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_delete_test::physical_snapshot;
|
||||
use super::artifact_repair_test::{RepairObject, finish_heal, start_heal};
|
||||
use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, s3_probe_client, unique_bucket, wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention,
|
||||
ObjectLockRetentionMode,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
const WRITES_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-writes";
|
||||
const INTERNAL_MESSAGE: &str = "We encountered an internal error, please try again.";
|
||||
|
||||
async fn seal_writes(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
|
||||
wait_until(Duration::from_secs(45), || async {
|
||||
let (status, response) = user.request(dist, 0, WRITES_ROUTE, json!({"version":1,"target":target}).to_string()).await?;
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed; retry the same identity") {
|
||||
return Ok(!confirmed);
|
||||
}
|
||||
assert!(confirmed, "offline configured receiver produced a complete receipt: {status} {response}");
|
||||
assert_eq!(status, StatusCode::OK, "combined writes seal: {response}");
|
||||
assert_eq!(serde_json::from_str::<Value>(&response)?, json!({"version":1,"target":target,"ordinary_put_sealed":true,"attempt_repairs_sealed":true,"all_writes_sealed":true}));
|
||||
Ok(true)
|
||||
}, "combined seal acknowledges both domains on every receiver").await
|
||||
}
|
||||
|
||||
async fn assert_put_closed(dist: &DistCluster, bucket: &str, key: &str) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let client = s3_probe_client(&dist.client(2)?);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(b"must-not-publish".to_vec().into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("combined seal admitted a new ordinary PUT");
|
||||
let service = error.as_service_error().ok_or("PUT was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"rejected PUT changed the managed physical namespace"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn lock_metadata_entrypoints(dist: &DistCluster, managed: &Value, object: &RepairObject, ordinary: &str) -> TestResult {
|
||||
let bucket = managed["bucket"].as_str().ok_or("bucket identity")?;
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
for bucket in [bucket, ordinary] {
|
||||
client
|
||||
.put_object_lock_configuration()
|
||||
.bucket(bucket)
|
||||
.object_lock_configuration(
|
||||
ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
for node in 0..4 {
|
||||
let output = match dist.client(node)?.get_object_lock_configuration().bucket(bucket).send().await {
|
||||
Ok(output) => output,
|
||||
Err(error)
|
||||
if error.as_service_error().and_then(ProvideErrorMetadata::code)
|
||||
== Some("ObjectLockConfigurationNotFoundError") =>
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if output
|
||||
.object_lock_configuration()
|
||||
.and_then(|configuration| configuration.object_lock_enabled())
|
||||
!= Some(&ObjectLockEnabled::Enabled)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"actual Object Lock configuration is enabled on all request nodes",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let control = RepairObject::put(dist, ordinary, "lock-control.bin".to_owned()).await?;
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.retain_until_date(aws_sdk_s3::primitives::DateTime::from_secs(chrono::Utc::now().timestamp() + 86400))
|
||||
.build();
|
||||
let hold = ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::On).build();
|
||||
client
|
||||
.put_object_retention()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.retention(retention.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
client
|
||||
.get_object_retention()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.send()
|
||||
.await?
|
||||
.retention()
|
||||
.and_then(|value| value.mode()),
|
||||
Some(&ObjectLockRetentionMode::Governance)
|
||||
);
|
||||
client
|
||||
.put_object_legal_hold()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.legal_hold(hold.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
client
|
||||
.get_object_legal_hold()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.send()
|
||||
.await?
|
||||
.legal_hold()
|
||||
.and_then(|value| value.status()),
|
||||
Some(&ObjectLockLegalHoldStatus::On)
|
||||
);
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let error = client
|
||||
.put_object_retention()
|
||||
.bucket(bucket)
|
||||
.key(&object.key)
|
||||
.version_id(&object.version)
|
||||
.retention(retention)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("managed Retention must reach and fail its unsupported metadata write");
|
||||
let service = error.as_service_error().ok_or("retention was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
|
||||
assert!(physical_snapshot(dist, bucket)? == before, "retention changed managed metadata");
|
||||
let error = client
|
||||
.put_object_legal_hold()
|
||||
.bucket(bucket)
|
||||
.key(&object.key)
|
||||
.version_id(&object.version)
|
||||
.legal_hold(hold)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("managed LegalHold must reach and fail its unsupported metadata write");
|
||||
let service = error.as_service_error().ok_or("legal hold was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some("Io error: artifact bucket does not support this write operation"));
|
||||
assert!(physical_snapshot(dist, bucket)? == before, "legal hold changed managed metadata");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_writes_seal_first_partial_retry_offline_and_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let ordinary = unique_bucket("writes-control");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary).await?;
|
||||
let (fresh, fresh_user) = create_artifact_attempt(&dist).await?;
|
||||
let fresh_bucket = fresh["bucket"].as_str().ok_or("fresh bucket")?;
|
||||
let fresh_prefix = fresh["attempt_prefix"].as_str().ok_or("fresh prefix")?;
|
||||
let first = RepairObject::put(&dist, fresh_bucket, format!("{fresh_prefix}existing.bin")).await?;
|
||||
lock_metadata_entrypoints(&dist, &fresh, &first, &ordinary).await?;
|
||||
seal_writes(&dist, &fresh_user, &fresh, true).await?;
|
||||
seal_writes(&dist, &fresh_user, &fresh, true).await?;
|
||||
assert_put_closed(&dist, fresh_bucket, &format!("{fresh_prefix}closed.bin")).await?;
|
||||
|
||||
let (partial, partial_user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = partial["bucket"].as_str().ok_or("partial bucket")?;
|
||||
let prefix = partial["attempt_prefix"].as_str().ok_or("partial prefix")?;
|
||||
let repaired = RepairObject::put(&dist, bucket, format!("{prefix}repair.bin")).await?;
|
||||
// Enter the combined operation with only its ordinary domain closed.
|
||||
confirm_seal(&dist, &partial_user, &json!({"version":1,"target":partial}).to_string(), &partial).await?;
|
||||
let part = repaired.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), bucket, 3)?;
|
||||
std::fs::remove_file(&part)?;
|
||||
let token = start_heal(&dist, bucket, &repaired.key, false).await?;
|
||||
repaired.assert_repaired(&dist, bucket).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
seal_writes(&dist, &partial_user, &partial, false).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
seal_writes(&dist, &partial_user, &partial, true).await?;
|
||||
}
|
||||
let ordinary_object = RepairObject::put(&dist, &ordinary, "repair-control.bin".to_owned()).await?;
|
||||
for restarted in [false, true] {
|
||||
if restarted {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
seal_writes(&dist, &partial_user, &partial, true).await?;
|
||||
assert_put_closed(&dist, bucket, &format!("{prefix}closed-{restarted}.bin")).await?;
|
||||
if part.exists() {
|
||||
std::fs::remove_file(&part)?;
|
||||
}
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let token = start_heal(&dist, bucket, &repaired.key, false).await?;
|
||||
finish_heal(&dist, &token, Some(&repaired.key)).await?;
|
||||
assert!(!part.exists(), "combined seal admitted a remote repair publication");
|
||||
assert!(physical_snapshot(&dist, bucket)? == before, "rejected repair changed metadata or shards");
|
||||
assert_exact_version(&dist.client(2)?, bucket, &repaired.key, &repaired.version, &repaired.body).await?;
|
||||
let ordinary_part = ordinary_object.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), &ordinary, 3)?;
|
||||
std::fs::remove_file(ordinary_part)?;
|
||||
let token = start_heal(&dist, &ordinary, &ordinary_object.key, false).await?;
|
||||
ordinary_object.assert_repaired(&dist, &ordinary).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -18,6 +18,13 @@
|
||||
//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
|
||||
//! each case starts four real `rustfs` processes.
|
||||
|
||||
mod artifact_delete_test;
|
||||
mod artifact_inventory_test;
|
||||
mod artifact_put_tail_test;
|
||||
mod artifact_repair_tail_test;
|
||||
mod artifact_repair_test;
|
||||
mod artifact_seal_test;
|
||||
mod artifact_writes_test;
|
||||
mod chaos_test;
|
||||
mod concurrency_stability_test;
|
||||
mod concurrent_data_movement_test;
|
||||
|
||||
@@ -541,6 +541,50 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_repair_rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_repair_write_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteMetadataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type InventoryArtifactAttemptStream =
|
||||
Pin<Box<dyn Stream<Item = Result<rustfs_protos::proto_gen::node_service::ArtifactInventoryFrame, Status>> + Send>>;
|
||||
async fn inventory_artifact_attempt(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactInventoryRequest>,
|
||||
) -> Result<Response<Self::InventoryArtifactAttemptStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn seal_artifact_attempt_repairs(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn seal_artifact_attempt_puts(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -15,12 +15,14 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption, Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskError, DiskOption, Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions, new_disk,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
verify_tonic_boot_epoch_response,
|
||||
set_tonic_canonical_body_digest, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
@@ -36,6 +38,11 @@ pub(crate) mod grpc_lock {
|
||||
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod artifact_rpc {
|
||||
pub(crate) use super::{DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
|
||||
}
|
||||
|
||||
/// Signing/transport surface used by the cross-process internode RPC signature
|
||||
/// acceptance tests (backlog#1327). The signing helpers are what let a test mint
|
||||
/// the one legitimately signed request an on-path attacker is assumed to have
|
||||
|
||||
@@ -382,6 +382,43 @@ pub mod config {
|
||||
with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
|
||||
with_server_config_write_lock,
|
||||
};
|
||||
|
||||
/// Observe persisted config bytes and their reader-associated metadata within a byte limit.
|
||||
pub async fn read_config_observation(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
/// Observe bounded config bytes while the caller retains the object's write owner.
|
||||
pub async fn read_config_observation_no_lock(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation_no_lock(api, file, max_bytes, guard).await
|
||||
}
|
||||
|
||||
/// Read config bytes and metadata without treating an empty payload as absent.
|
||||
pub async fn read_config_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
|
||||
}
|
||||
|
||||
pub async fn read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
}
|
||||
|
||||
pub mod storageclass {
|
||||
@@ -431,16 +468,19 @@ pub mod disk {
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
|
||||
ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
is_quota_mutation_fence_path, new_disk, validate_batch_read_version_item_count,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, LegacyFileStream, LegacyFileWrite,
|
||||
LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete, LegacyPathDeleteOperation,
|
||||
LegacyRenameData, LegacyRenameFile, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path,
|
||||
new_disk, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
@@ -635,6 +675,7 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,120 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
|
||||
use crate::config::com::read_config_observation;
|
||||
use crate::object_api::WriteCompletion;
|
||||
|
||||
impl ECStore {
|
||||
/// Permanently retain this managed bucket's versions for historical restores.
|
||||
/// Lifecycle WRITE drains earlier exact deletes. The detached owner keeps
|
||||
/// lifecycle -> metadata transaction locks until every metadata write drains.
|
||||
pub async fn retain_artifact_backups(self: Arc<Self>, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<()> {
|
||||
let bucket = bucket.to_owned();
|
||||
let base_prefix = base_prefix.to_owned();
|
||||
let ctx = Arc::clone(&self.ctx);
|
||||
ctx.run_detached_mutation(async move {
|
||||
require_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact backup retention requires namespace locking"));
|
||||
}
|
||||
let lifecycle = self.acquire_bucket_lifecycle_write_lock(&bucket).await?;
|
||||
let transaction = metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, &bucket).await?;
|
||||
let mut metadata = self.backup_retention_metadata(&bucket, incarnation, &base_prefix).await?;
|
||||
let check = || {
|
||||
if lifecycle.is_lock_lost() || transaction.is_lock_lost() {
|
||||
Err(Error::other("artifact backup retention fence was lost"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
check()?;
|
||||
// Even an idempotent retry rewrites with TailDrained: an earlier
|
||||
// interrupted call may have reached only some metadata replicas.
|
||||
metadata.artifact_backup_retained = true;
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
opts.add_namespace_lock_guard(&transaction);
|
||||
metadata.save_with_store_options(self.clone(), &opts).await?;
|
||||
check()?;
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
sys.set_validated(metadata).await?;
|
||||
check()
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact backup retention owner failed"))?
|
||||
}
|
||||
|
||||
/// Observe the immutable bucket identity and retention in one locked read.
|
||||
pub async fn inspect_artifact_backup_retention(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<bool> {
|
||||
require_put_seal_topology(self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact backup retention requires namespace locking"));
|
||||
}
|
||||
let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let metadata = self.backup_retention_metadata(bucket, incarnation, base_prefix).await?;
|
||||
if lifecycle.is_lock_lost() {
|
||||
return Err(Error::other("artifact backup retention observation fence was lost"));
|
||||
}
|
||||
Ok(metadata.artifact_backup_retained)
|
||||
}
|
||||
|
||||
async fn backup_retention_metadata(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<BucketMetadata> {
|
||||
if incarnation.is_nil() || !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(base_prefix) {
|
||||
return Err(Error::other("artifact backup retention identity is invalid"));
|
||||
}
|
||||
let mut metadata = self.read_backup_retention_metadata(bucket).await?;
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let (sidecar, _) =
|
||||
read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
|
||||
metadata.parse_all_configs()?;
|
||||
if metadata.bucket_incarnation_id != incarnation
|
||||
|| Uuid::from_slice(&sidecar).ok() != Some(incarnation)
|
||||
|| metadata.artifact_base_prefix.as_deref() != Some(base_prefix)
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| !metadata.versioning_config.as_ref().is_some_and(|versioning| {
|
||||
versioning.enabled()
|
||||
&& !versioning.exclude_folders.unwrap_or(false)
|
||||
&& versioning.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
})
|
||||
{
|
||||
return Err(Error::other("artifact backup retention identity mismatch"));
|
||||
}
|
||||
metadata.bucket_incarnation_sidecar = true;
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
/// Caller must hold the bucket lifecycle fence. An unreadable or missing
|
||||
/// record cannot prove that historical backup protection was never enabled.
|
||||
pub(crate) async fn require_no_artifact_backup_retention(&self, bucket: &str) -> Result<()> {
|
||||
if self.read_backup_retention_metadata(bucket).await?.artifact_backup_retained {
|
||||
return Err(Error::other("artifact versions are retained for backups"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_backup_retention_metadata(&self, bucket: &str) -> Result<BucketMetadata> {
|
||||
if bucket.is_empty()
|
||||
|| super::super::utils::has_bad_path_component(bucket)
|
||||
|| super::super::utils::is_meta_bucketname(bucket)
|
||||
{
|
||||
return Err(Error::other("artifact backup retention bucket is invalid"));
|
||||
}
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if store.id != self.id {
|
||||
return Err(Error::other("artifact backup retention belongs to another store"));
|
||||
}
|
||||
let (bytes, _) = read_config_observation(store, &BucketMetadata::new(bucket).save_file_path(), 64 * 1024 * 1024).await?;
|
||||
let metadata = BucketMetadata::decode_file(&bytes)?;
|
||||
if metadata.name != bucket {
|
||||
return Err(Error::other("artifact backup retention bucket mismatch"));
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
|
||||
/// In-process authority for exactly one drained managed version. No caller can
|
||||
/// construct this capability or serialize it into a disk/RPC request.
|
||||
#[doc(hidden)]
|
||||
pub struct ArtifactDeleteTarget {
|
||||
store: Uuid,
|
||||
record: ArtifactVersionRetirementRecord,
|
||||
lifecycle: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
attempt: WriteCommitGuard,
|
||||
repair: WriteCommitGuard,
|
||||
}
|
||||
|
||||
impl ArtifactDeleteTarget {
|
||||
fn check_live(&self) -> Result<()> {
|
||||
if self.lifecycle.is_lock_lost() || self.attempt.is_lock_lost() || self.repair.is_lock_lost() {
|
||||
return Err(Error::other("artifact exact delete owned fence was lost"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn lifecycle(&self) -> Arc<rustfs_lock::NamespaceLockGuard> {
|
||||
Arc::clone(&self.lifecycle)
|
||||
}
|
||||
|
||||
pub(crate) fn validate(&self, store: &ECStore, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
if self.store != store.id
|
||||
|| bucket != self.record.bucket
|
||||
|| rustfs_utils::path::encode_dir_object(object) != self.record.object
|
||||
|| opts.expected_bucket_incarnation_id != Some(self.record.incarnation_id)
|
||||
|| opts.version_id.as_deref() != Some(self.record.version_id.to_string().as_str())
|
||||
|| !opts.versioned
|
||||
|| opts.version_suspended
|
||||
|| opts.delete_prefix
|
||||
|| opts.delete_marker
|
||||
|| opts.data_movement
|
||||
|| opts.lifecycle_delete_all.is_some()
|
||||
|| self.lifecycle.is_lock_lost()
|
||||
|| self.attempt.is_lock_lost()
|
||||
|| self.repair.is_lock_lost()
|
||||
{
|
||||
return Err(Error::other("artifact exact delete target or owned fence mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Read-only capability observation; each mutation revalidates the topology.
|
||||
pub async fn supports_artifact_exact_delete(&self) -> bool {
|
||||
require_local_put_seal_topology(self).await.is_ok() && !self.ctx.lock_manager().is_disabled()
|
||||
}
|
||||
|
||||
/// Delete an exact version in a fully sealed attempt. This authenticates the
|
||||
/// storage identity and drains writers; the caller still owns GC eligibility
|
||||
/// and cleanup IAM. Distributed receiver deletion is not yet supported.
|
||||
pub async fn delete_artifact_object_version(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
expected_incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
) -> Result<()> {
|
||||
require_local_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact exact deletion requires namespace locking"));
|
||||
}
|
||||
let lifecycle = Arc::new(self.acquire_bucket_lifecycle_read_lock(bucket).await?);
|
||||
self.require_no_artifact_backup_retention(bucket).await?;
|
||||
let put = self
|
||||
.inventory_artifact_record(bucket, expected_incarnation, attempt_prefix)
|
||||
.await?;
|
||||
let repair = ArtifactRepairSealRecord::new(&put);
|
||||
let record = ArtifactVersionRetirementRecord::new(&put, object, version)?;
|
||||
let object = object.to_owned();
|
||||
let store = Arc::clone(&self);
|
||||
// Lifecycle -> attempt -> repair seal -> metadata transaction -> object.
|
||||
// The spawned owner, and the disk deletion tail's typed capability, keep
|
||||
// these locks after cancellation of the HTTP caller.
|
||||
with_config_object_write_lock(self, put.path(), move |attempt| async move {
|
||||
if !put.exists(Arc::clone(&store), &attempt).await? {
|
||||
return Err(Error::other("artifact exact deletion requires sealed PUTs"));
|
||||
}
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
|
||||
let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
|
||||
if !repair.exists(Arc::clone(&store), &repair_owner).await? {
|
||||
return Err(Error::other("artifact exact deletion requires sealed repairs"));
|
||||
}
|
||||
let receivers = store.capture_artifact_receivers().await?;
|
||||
store
|
||||
.close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Put, &receivers)
|
||||
.await?;
|
||||
store
|
||||
.close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Repair, &receivers)
|
||||
.await?;
|
||||
store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
|
||||
let target = Arc::new(ArtifactDeleteTarget {
|
||||
store: store.id,
|
||||
record,
|
||||
lifecycle,
|
||||
attempt,
|
||||
repair: repair_owner,
|
||||
});
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.to_string()),
|
||||
expected_bucket_incarnation_id: Some(expected_incarnation),
|
||||
artifact_delete_target: Some(Arc::clone(&target)),
|
||||
delete_mutation_owner: Some(target.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_write_commit_guard(&target.attempt);
|
||||
opts.add_write_commit_guard(&target.repair);
|
||||
// These owners fence seal records, not the object. Ordinary DELETE
|
||||
// must still acquire and retain its exact object WRITE owner.
|
||||
opts.no_lock = false;
|
||||
target.check_live()?;
|
||||
// Preserve the normal exact DELETE's Object Lock, quorum, journal,
|
||||
// cache and capacity behavior. Its success is logical deletion, not
|
||||
// a claim that every offline shard has been physically erased.
|
||||
match store.delete_object(&put.bucket, &object, opts).await {
|
||||
Ok(info) => {
|
||||
// Match the S3 request layer's committed-generation quota
|
||||
// delta. A strict absence retry must not subtract twice.
|
||||
if !info.name.is_empty()
|
||||
&& !info.delete_marker
|
||||
&& let Ok(size) = crate::data_usage::quota_object_size(&info)
|
||||
{
|
||||
crate::data_usage::record_bucket_object_delete_memory(&put.bucket, size, false).await;
|
||||
}
|
||||
}
|
||||
Err(Error::VersionNotFound(bucket, object, version))
|
||||
if bucket == target.record.bucket
|
||||
&& rustfs_utils::path::encode_dir_object(&object) == target.record.object
|
||||
&& version == target.record.version_id.to_string() => {}
|
||||
Err(Error::ObjectNotFound(bucket, object))
|
||||
if bucket == target.record.bucket
|
||||
&& rustfs_utils::path::encode_dir_object(&object) == target.record.object => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
|
||||
store
|
||||
.inventory_artifact_record(&put.bucket, expected_incarnation, &put.attempt_prefix)
|
||||
.await?;
|
||||
target.check_live()
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion, Disk};
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase;
|
||||
use tokio::{sync::mpsc, time::Instant};
|
||||
|
||||
/// Physical observations retain their member. Replicas are not extra business
|
||||
/// versions and this stream makes no quorum-readability or GC claim.
|
||||
#[derive(Debug)]
|
||||
pub enum ArtifactAttemptInventoryFrame {
|
||||
Observations {
|
||||
endpoint: String,
|
||||
disk_id: Uuid,
|
||||
versions: Vec<ArtifactInventoryVersion>,
|
||||
},
|
||||
Complete {
|
||||
members: usize,
|
||||
observations: u64,
|
||||
},
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Inventory one fully sealed attempt across every configured receiver.
|
||||
/// Earlier output is staging only; only the final frame plus Ok is complete.
|
||||
pub async fn inventory_artifact_attempt(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
prefix: &str,
|
||||
deadline: Instant,
|
||||
output: mpsc::Sender<ArtifactAttemptInventoryFrame>,
|
||||
) -> Result<()> {
|
||||
tokio::time::timeout_at(deadline, Box::pin(async move {
|
||||
require_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() { return Err(Error::other("artifact inventory requires namespace locking")); }
|
||||
let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let put = self.inventory_artifact_record(bucket, incarnation, prefix).await?;
|
||||
let repair = ArtifactRepairSealRecord::new(&put);
|
||||
// Lifecycle -> attempt -> repair record. Receivers do not reenter
|
||||
// distributed locks; actual two-domain disk owners guard their reads.
|
||||
let store = self.clone();
|
||||
let bucket = bucket.to_owned();
|
||||
let prefix = prefix.to_owned();
|
||||
with_config_object_write_lock(self, put.path(), move |attempt| async move {
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
if !put.exists(store.clone(), &attempt).await? { return Err(Error::other("artifact inventory requires PUT seal record")); }
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
|
||||
let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
|
||||
if !repair.exists(store.clone(), &repair_owner).await? { return Err(Error::other("artifact inventory requires repair seal record")); }
|
||||
let receivers = store.capture_artifact_receivers().await?;
|
||||
let mut observations = 0u64;
|
||||
for (endpoint, (disk, expected)) in &receivers {
|
||||
if lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
|
||||
return Err(Error::other("artifact inventory fence was lost"));
|
||||
}
|
||||
let (sender, mut frames) = mpsc::channel(2);
|
||||
let scan = async {
|
||||
match disk.as_ref() {
|
||||
Disk::Local(local) => local.get_disk().inventory_artifact_attempt(*expected, &bucket, &prefix, put.identity(), deadline, sender).await,
|
||||
Disk::Remote(remote) => remote.inventory_artifact_attempt(*expected, (&bucket, incarnation), &prefix, ArtifactInventoryPhase::ArtifactInventoryScan, deadline, sender).await,
|
||||
}
|
||||
};
|
||||
tokio::pin!(scan);
|
||||
let mut sequence = 0;
|
||||
let mut count = 0u64;
|
||||
let mut terminal = false;
|
||||
let mut ended = false;
|
||||
let mut done = false;
|
||||
while !ended || !done {
|
||||
tokio::select! {
|
||||
result = &mut scan, if !done => { result.map_err(Error::other)?; done = true; },
|
||||
frame = frames.recv(), if !ended => match frame {
|
||||
Some(ArtifactInventoryFrame::Entries { sequence: actual, versions }) if !terminal && actual == sequence => {
|
||||
count = count.checked_add(u64::try_from(versions.len()).map_err(Error::other)?).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
|
||||
output.send(ArtifactAttemptInventoryFrame::Observations { endpoint: endpoint.clone(), disk_id: *expected, versions }).await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
sequence += 1;
|
||||
}
|
||||
Some(ArtifactInventoryFrame::Complete { disk_id, incarnation: actual, attempt_prefix, entries })
|
||||
if !terminal && disk_id == *expected && actual == incarnation && attempt_prefix == prefix && entries == count => terminal = true,
|
||||
Some(_) => return Err(Error::other("artifact inventory member stream is invalid")),
|
||||
None => ended = true,
|
||||
},
|
||||
}
|
||||
}
|
||||
if !terminal { return Err(Error::other("artifact inventory member is incomplete")); }
|
||||
observations = observations.checked_add(count).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
|
||||
}
|
||||
store.verify_inventory_members(&put, &receivers, deadline).await?;
|
||||
let current = store.artifact_inventory_receiver_identity(&bucket, incarnation, &prefix).await?;
|
||||
if current.receiver_key != put.identity().receiver_key || lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
|
||||
return Err(Error::other("artifact inventory authority changed"));
|
||||
}
|
||||
output.send(ArtifactAttemptInventoryFrame::Complete { members: receivers.len(), observations }).await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
Ok(())
|
||||
}).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
|
||||
}).await?
|
||||
})).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
|
||||
}
|
||||
|
||||
pub(super) async fn inventory_artifact_record(&self, bucket: &str, expected: Uuid, prefix: &str) -> Result<PutSealRecord> {
|
||||
use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
|
||||
use crate::config::com::read_config_observation;
|
||||
if expected.is_nil()
|
||||
|| crate::bucket::utils::has_bad_path_component(bucket)
|
||||
|| bucket.is_empty()
|
||||
|| crate::bucket::utils::is_meta_bucketname(bucket)
|
||||
{
|
||||
return Err(Error::other("artifact inventory authority is invalid"));
|
||||
}
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if store.id != self.id {
|
||||
return Err(Error::other("artifact inventory authority belongs to another store"));
|
||||
}
|
||||
// Do not enter metadata_sys's lazy bucket READ/migration path: the
|
||||
// coordinator already owns lifecycle READ and a writer may be queued.
|
||||
// Observation reads bypass cache and suppress physical read repair.
|
||||
let path = BucketMetadata::new(bucket).save_file_path();
|
||||
let (bytes, _) = read_config_observation(store.clone(), &path, 64 * 1024 * 1024).await?;
|
||||
let mut metadata = BucketMetadata::decode_file(&bytes)?;
|
||||
let (sidecar, _) =
|
||||
read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
|
||||
if metadata.name != bucket
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| metadata.bucket_incarnation_id != expected
|
||||
|| Uuid::from_slice(&sidecar).ok() != Some(expected)
|
||||
{
|
||||
return Err(Error::other("artifact inventory incarnation mismatch"));
|
||||
}
|
||||
metadata.parse_all_configs()?;
|
||||
if !metadata.versioning_config.as_ref().is_some_and(|config| {
|
||||
config.enabled()
|
||||
&& !config.exclude_folders.unwrap_or(false)
|
||||
&& config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
}) {
|
||||
return Err(Error::other("artifact inventory requires managed versioning"));
|
||||
}
|
||||
PutSealRecord::new(&metadata, prefix)
|
||||
}
|
||||
|
||||
pub(crate) async fn artifact_inventory_receiver_identity(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: Uuid,
|
||||
prefix: &str,
|
||||
) -> Result<ArtifactPutIdentity> {
|
||||
Ok(self.inventory_artifact_record(bucket, expected, prefix).await?.identity())
|
||||
}
|
||||
|
||||
async fn verify_inventory_members(
|
||||
&self,
|
||||
record: &PutSealRecord,
|
||||
captured: &ArtifactReceiverSnapshot,
|
||||
deadline: Instant,
|
||||
) -> Result<()> {
|
||||
for final_pass in [false, true] {
|
||||
let current = self.artifact_receiver_inventory().await?;
|
||||
if current.len() != captured.len()
|
||||
|| current
|
||||
.iter()
|
||||
.any(|(endpoint, disk)| !captured.get(endpoint).is_some_and(|(old, _)| Arc::ptr_eq(old, disk)))
|
||||
{
|
||||
return Err(Error::other("artifact inventory membership changed"));
|
||||
}
|
||||
if final_pass {
|
||||
for (disk, expected) in captured.values() {
|
||||
if disk.cached_disk_id().await != Some(*expected) {
|
||||
return Err(Error::other("artifact inventory expected identity changed"));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
for (disk, expected) in captured.values() {
|
||||
if disk.cached_disk_id().await != Some(*expected) {
|
||||
return Err(Error::other("artifact inventory expected identity changed"));
|
||||
}
|
||||
match disk.as_ref() {
|
||||
Disk::Local(local) => {
|
||||
let _verification = local
|
||||
.get_disk()
|
||||
.verify_artifact_inventory_closed(*expected, record.identity(), deadline)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
}
|
||||
Disk::Remote(remote) => {
|
||||
let (output, _unused) = mpsc::channel(1);
|
||||
remote
|
||||
.inventory_artifact_attempt(
|
||||
*expected,
|
||||
(&record.bucket, record.incarnation_id),
|
||||
&record.attempt_prefix,
|
||||
ArtifactInventoryPhase::ArtifactInventoryVerify,
|
||||
deadline,
|
||||
output,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
|
||||
/// Shares only reads started after a caller arrives; retains no idle results.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct OrdinaryClassificationRegistry {
|
||||
gates: Mutex<HashMap<String, Weak<Gate>>>,
|
||||
#[cfg(test)]
|
||||
after_unregister: Option<std::sync::Barrier>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Gate {
|
||||
started_generation: AtomicU64,
|
||||
completed: tokio::sync::Mutex<Option<CompletedRead>>,
|
||||
}
|
||||
|
||||
struct CompletedRead {
|
||||
generation: u64,
|
||||
allowed: bool,
|
||||
}
|
||||
|
||||
struct Registration<'a> {
|
||||
registry: &'a OrdinaryClassificationRegistry,
|
||||
bucket: &'a str,
|
||||
gate: Option<Arc<Gate>>,
|
||||
observed_generation: u64,
|
||||
}
|
||||
|
||||
impl OrdinaryClassificationRegistry {
|
||||
pub(crate) async fn classify(&self, bucket: &str, read: impl Future<Output = bool>) -> bool {
|
||||
self.register(bucket).classify(read).await
|
||||
}
|
||||
|
||||
fn register<'a>(&'a self, bucket: &'a str) -> Registration<'a> {
|
||||
// Callers requiring physical admission must register after acquiring it.
|
||||
// Release the registry lock before awaiting the gate, then acquire the
|
||||
// existing metadata locks.
|
||||
let mut gates = self.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let gate = gates.get(bucket).and_then(Weak::upgrade).unwrap_or_else(|| {
|
||||
let gate = Arc::new(Gate::default());
|
||||
gates.insert(bucket.to_owned(), Arc::downgrade(&gate));
|
||||
gate
|
||||
});
|
||||
let observed_generation = gate.started_generation.load(Ordering::Acquire);
|
||||
Registration {
|
||||
registry: self,
|
||||
bucket,
|
||||
gate: Some(gate),
|
||||
observed_generation,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Registration<'_> {
|
||||
async fn classify(self, read: impl Future<Output = bool>) -> bool {
|
||||
let Some(gate) = self.gate.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
let mut completed = gate.completed.lock().await;
|
||||
// Only a read started after this caller registered can authorize it.
|
||||
// Arrivals during a read therefore wait for the next fresh read.
|
||||
if let Some(result) = completed
|
||||
.as_ref()
|
||||
.filter(|result| result.generation > self.observed_generation)
|
||||
{
|
||||
return result.allowed;
|
||||
}
|
||||
let Ok(previous) = gate
|
||||
.started_generation
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
// Keep the gate through the original bounded metadata read. Cancellation
|
||||
// releases it without publishing a result; another caller can read again.
|
||||
let allowed = read.await;
|
||||
*completed = Some(CompletedRead {
|
||||
generation: previous + 1,
|
||||
allowed,
|
||||
});
|
||||
allowed
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Registration<'_> {
|
||||
fn drop(&mut self) {
|
||||
let Some(gate) = self.gate.take() else {
|
||||
return;
|
||||
};
|
||||
let mut gates = self.registry.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Strong references are acquired and released under this registry lock,
|
||||
// so concurrent final callers cannot both miss the last reference.
|
||||
if Arc::strong_count(&gate) == 1
|
||||
&& gates
|
||||
.get(self.bucket)
|
||||
.is_some_and(|registered| registered.ptr_eq(&Arc::downgrade(&gate)))
|
||||
{
|
||||
gates.remove(self.bucket);
|
||||
}
|
||||
drop(gate);
|
||||
drop(gates);
|
||||
#[cfg(test)]
|
||||
if let Some(barrier) = &self.registry.after_unregister {
|
||||
barrier.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures::poll;
|
||||
use std::task::Poll;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
fn assert_empty(registry: &OrdinaryClassificationRegistry) {
|
||||
assert!(registry.gates.lock().expect("classification registry").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_arrivals_share_only_the_next_fail_closed_read() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
let (release, waiting) = oneshot::channel();
|
||||
let mut first = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
waiting.await.expect("release first read");
|
||||
true
|
||||
}));
|
||||
assert!(poll!(first.as_mut()).is_pending());
|
||||
let mut followers = (0..16)
|
||||
.map(|_| {
|
||||
Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
false
|
||||
}))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for follower in &mut followers {
|
||||
assert!(poll!(follower.as_mut()).is_pending());
|
||||
}
|
||||
release.send(()).expect("first reader still alive");
|
||||
assert_eq!(poll!(first.as_mut()), Poll::Ready(true));
|
||||
for follower in &mut followers {
|
||||
assert_eq!(poll!(follower.as_mut()), Poll::Ready(false));
|
||||
}
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 2);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successive_waves_always_read_again() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
for expected in [true, false, true] {
|
||||
assert_eq!(
|
||||
registry
|
||||
.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
expected
|
||||
})
|
||||
.await,
|
||||
expected
|
||||
);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_leader_does_not_publish_or_strand_waiters() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
let old_registration = registry.register("bucket");
|
||||
assert!(
|
||||
registry
|
||||
.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
true
|
||||
})
|
||||
.await
|
||||
);
|
||||
let mut leader = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
std::future::pending::<bool>().await
|
||||
}));
|
||||
assert!(poll!(leader.as_mut()).is_pending());
|
||||
let mut old_waiter =
|
||||
Box::pin(old_registration.classify(async { panic!("old waiter must reuse the completed post-registration read") }));
|
||||
assert!(poll!(old_waiter.as_mut()).is_pending());
|
||||
let mut late_waiter = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
false
|
||||
}));
|
||||
assert!(poll!(late_waiter.as_mut()).is_pending());
|
||||
drop(leader);
|
||||
assert_eq!(poll!(old_waiter.as_mut()), Poll::Ready(true));
|
||||
assert_eq!(poll!(late_waiter.as_mut()), Poll::Ready(false));
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 3);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_follower_does_not_retain_the_gate() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let (release, waiting) = oneshot::channel();
|
||||
let mut leader = Box::pin(registry.classify("bucket", async {
|
||||
waiting.await.expect("release leader");
|
||||
true
|
||||
}));
|
||||
assert!(poll!(leader.as_mut()).is_pending());
|
||||
let mut follower = Box::pin(registry.classify("bucket", async { panic!("canceled follower must not read") }));
|
||||
assert!(poll!(follower.as_mut()).is_pending());
|
||||
drop(follower);
|
||||
assert_eq!(registry.gates.lock().expect("classification registry").len(), 1);
|
||||
release.send(()).expect("leader still alive");
|
||||
assert_eq!(poll!(leader.as_mut()), Poll::Ready(true));
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_final_registrations_remove_the_gate() {
|
||||
let registry = OrdinaryClassificationRegistry {
|
||||
after_unregister: Some(std::sync::Barrier::new(2)),
|
||||
..Default::default()
|
||||
};
|
||||
let first = registry.register("bucket");
|
||||
let second = registry.register("bucket");
|
||||
// Both destructors reach the unlocked barrier before either returns.
|
||||
// Moving the Arc release after this barrier leaves a stale registry key.
|
||||
std::thread::scope(|scope| {
|
||||
scope.spawn(move || drop(first));
|
||||
scope.spawn(move || drop(second));
|
||||
});
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buckets_and_stores_do_not_share_gates() {
|
||||
let first_store = OrdinaryClassificationRegistry::default();
|
||||
let second_store = OrdinaryClassificationRegistry::default();
|
||||
let mut blocked = Box::pin(first_store.classify("first", std::future::pending()));
|
||||
assert!(poll!(blocked.as_mut()).is_pending());
|
||||
assert_eq!(poll!(Box::pin(first_store.classify("second", async { true }))), Poll::Ready(true));
|
||||
assert_eq!(poll!(Box::pin(second_store.classify("first", async { false }))), Poll::Ready(false));
|
||||
assert_empty(&second_store);
|
||||
drop(blocked);
|
||||
assert_empty(&first_store);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generation_exhaustion_fails_closed_without_reading() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let registration = registry.register("bucket");
|
||||
let gate = registration.gate.as_ref().expect("registered gate");
|
||||
gate.started_generation.store(u64::MAX, Ordering::Release);
|
||||
*gate.completed.lock().await = Some(CompletedRead {
|
||||
generation: u64::MAX,
|
||||
allowed: true,
|
||||
});
|
||||
assert!(
|
||||
!registry
|
||||
.classify("bucket", async { panic!("exhausted generation must deny without reading") })
|
||||
.await
|
||||
);
|
||||
assert_eq!(gate.started_generation.load(Ordering::Acquire), u64::MAX);
|
||||
drop(registration);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
}
|
||||
@@ -890,6 +890,7 @@ pub(crate) async fn cleanup_free_version_exact(
|
||||
let bucket_guard = api
|
||||
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
|
||||
.await
|
||||
.map(Arc::new)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
|
||||
let local_object = encode_dir_object(&oi.name);
|
||||
@@ -958,7 +959,7 @@ pub(crate) async fn cleanup_free_version_exact(
|
||||
};
|
||||
delete_request.set_tier_free_version();
|
||||
if let Err(err) = set
|
||||
.delete_object_version(&oi.bucket, &local_object, &delete_request, false)
|
||||
.delete_object_version_with_lifecycle_owner(&oi.bucket, &local_object, &delete_request, Arc::clone(&bucket_guard))
|
||||
.await
|
||||
&& first_error.is_none()
|
||||
{
|
||||
@@ -8496,6 +8497,104 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn artifact_delete_entrypoints_free_version_cleaner_completes_with_queued_lifecycle_writer() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("free-version-queued-writer-{}", Uuid::new_v4());
|
||||
let object = "free-version";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let (backend, identity) = register_recovery_mock_tier(&ecstore).await;
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity)).await;
|
||||
let oi = list_tier_free_versions(Arc::clone(&ecstore), 100, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("list real persisted free version")
|
||||
.items
|
||||
.into_iter()
|
||||
.find(|oi| oi.bucket == bucket)
|
||||
.expect("free-version task must address this fixture");
|
||||
assert!(oi.transitioned_object.free_version);
|
||||
backend
|
||||
.set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
|
||||
.await;
|
||||
let remote = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("remote fixture lease");
|
||||
remote
|
||||
.put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
|
||||
.await
|
||||
.expect("seed exact remote object");
|
||||
drop(remote);
|
||||
|
||||
let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
|
||||
let cleanup_barrier = Arc::clone(&barrier);
|
||||
let cleanup_store = Arc::clone(&ecstore);
|
||||
let cleanup_oi = oi.clone();
|
||||
let mut cleanup = tokio::spawn(async move {
|
||||
super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
|
||||
.scope(cleanup_barrier, async move {
|
||||
super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(10), barrier.arrived.notified())
|
||||
.await
|
||||
.expect("cleaner owns lifecycle READ and has completed remote DELETE");
|
||||
assert!(!backend.contains(&oi.transitioned_object.name).await);
|
||||
for disk in &disk_paths {
|
||||
assert!(
|
||||
disk.join(&bucket).join(object).join(STORAGE_FORMAT_FILE).is_file(),
|
||||
"local exact cleanup must still be pending at the boundary"
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve first so the pending poll is the actual local lock wait.
|
||||
// A later READ would normally be admitted alongside the cleaner's READ;
|
||||
// its pending state independently proves this WRITE is now queued.
|
||||
let lifecycle = ecstore
|
||||
.new_ns_lock(&bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||
.await
|
||||
.expect("lifecycle namespace");
|
||||
let mut writer = Box::pin(lifecycle.get_write_lock(StdDuration::from_secs(30)));
|
||||
let writer_queued = futures::poll!(&mut writer).is_pending();
|
||||
let mut later_reader = Box::pin(lifecycle.get_read_lock(StdDuration::from_secs(30)));
|
||||
let later_reader_blocked = futures::poll!(&mut later_reader).is_pending();
|
||||
drop(later_reader);
|
||||
barrier.release.notify_one();
|
||||
assert!(writer_queued, "cleaner must retain the real lifecycle READ owner");
|
||||
assert!(later_reader_blocked, "queued lifecycle WRITE must block a later READ");
|
||||
|
||||
// Keep the same WRITE future queued, without cancelling/retrying it.
|
||||
// Re-entering READ here would deadlock behind this waiting writer.
|
||||
let finished = tokio::time::timeout(StdDuration::from_secs(10), &mut cleanup).await;
|
||||
if finished.is_err() {
|
||||
cleanup.abort();
|
||||
drop(writer);
|
||||
let _ = tokio::time::timeout(StdDuration::from_secs(5), &mut cleanup).await;
|
||||
panic!("exact local free-version cleanup re-entered lifecycle READ behind the queued writer");
|
||||
}
|
||||
assert!(
|
||||
finished
|
||||
.expect("bounded cleaner completion")
|
||||
.expect("cleaner task")
|
||||
.expect("exact local cleanup")
|
||||
);
|
||||
let writer = tokio::time::timeout(StdDuration::from_secs(10), writer)
|
||||
.await
|
||||
.expect("queued lifecycle WRITE proceeds after real cleanup drains")
|
||||
.expect("lifecycle WRITE acquired");
|
||||
for disk in &disk_paths {
|
||||
assert!(!disk.join(&bucket).join(object).exists(), "free version remains on a real local disk");
|
||||
}
|
||||
assert_eq!(backend.remove_count().await, 1, "cleaner must delete the exact remote tuple once");
|
||||
drop(writer);
|
||||
ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("remove empty fixture bucket");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
|
||||
@@ -437,6 +437,8 @@ pub struct BucketMetadata {
|
||||
pub created: OffsetDateTime,
|
||||
pub lock_enabled: bool, // While marked as unused, it may need to be retained
|
||||
pub bucket_incarnation_id: Uuid,
|
||||
pub artifact_base_prefix: Option<String>,
|
||||
pub artifact_backup_retained: bool,
|
||||
pub(crate) bucket_incarnation_sidecar: bool,
|
||||
pub(crate) bucket_creation_committed: bool,
|
||||
pub policy_config_json: Vec<u8>,
|
||||
@@ -512,6 +514,8 @@ impl Default for BucketMetadata {
|
||||
created: OffsetDateTime::UNIX_EPOCH,
|
||||
lock_enabled: Default::default(),
|
||||
bucket_incarnation_id: Uuid::nil(),
|
||||
artifact_base_prefix: None,
|
||||
artifact_backup_retained: false,
|
||||
bucket_incarnation_sidecar: false,
|
||||
bucket_creation_committed: false,
|
||||
policy_config_json: Default::default(),
|
||||
@@ -607,7 +611,7 @@ impl BucketMetadata {
|
||||
/// legacy migration path, which mints the sidecar first; it must not be
|
||||
/// rewritten by this migration.
|
||||
pub(crate) fn needs_bucket_creation_commit(&self) -> bool {
|
||||
self.lock_enabled
|
||||
(self.lock_enabled || self.artifact_base_prefix.is_some())
|
||||
&& !self.bucket_creation_committed
|
||||
&& self.bucket_incarnation_sidecar
|
||||
&& !self.bucket_incarnation_id.is_nil()
|
||||
@@ -704,6 +708,22 @@ impl BucketMetadata {
|
||||
self.bucket_incarnation_id =
|
||||
Uuid::from_slice(&bytes).map_err(|err| Error::other(format!("invalid BucketIncarnationID: {err}")))?;
|
||||
}
|
||||
"ArtifactBackupRetained" => {
|
||||
if self.artifact_backup_retained || !rmp::decode::read_bool(rd)? {
|
||||
return Err(Error::other("invalid artifact backup retention marker"));
|
||||
}
|
||||
self.artifact_backup_retained = true;
|
||||
}
|
||||
"ArtifactBasePrefix" => {
|
||||
if self.artifact_base_prefix.is_some() {
|
||||
return Err(Error::other("duplicate artifact namespace"));
|
||||
}
|
||||
let prefix = read_msgp_str(rd)?;
|
||||
if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(&prefix) {
|
||||
return Err(Error::other("invalid artifact namespace"));
|
||||
}
|
||||
self.artifact_base_prefix = Some(prefix);
|
||||
}
|
||||
"PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
|
||||
"NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
|
||||
"LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
|
||||
@@ -766,7 +786,7 @@ impl BucketMetadata {
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (22)
|
||||
let map_len: u32 = 47;
|
||||
let map_len: u32 = 47 + u32::from(self.artifact_base_prefix.is_some()) + u32::from(self.artifact_backup_retained);
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -780,6 +800,15 @@ impl BucketMetadata {
|
||||
rmp::encode::write_bool(wr, self.lock_enabled)?;
|
||||
|
||||
write_bin_field(wr, "BucketIncarnationID", self.bucket_incarnation_id.as_bytes())?;
|
||||
if let Some(prefix) = &self.artifact_base_prefix {
|
||||
rmp::encode::write_str(wr, "ArtifactBasePrefix")?;
|
||||
rmp::encode::write_str(wr, prefix)?;
|
||||
}
|
||||
|
||||
if self.artifact_backup_retained {
|
||||
rmp::encode::write_str(wr, "ArtifactBackupRetained")?;
|
||||
rmp::encode::write_bool(wr, true)?;
|
||||
}
|
||||
|
||||
write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?;
|
||||
write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?;
|
||||
@@ -880,13 +909,28 @@ impl BucketMetadata {
|
||||
}
|
||||
|
||||
match version {
|
||||
BUCKET_METADATA_VERSION => {}
|
||||
BUCKET_METADATA_VERSION | 2 => {}
|
||||
_ => return Err(Error::other("read_bucket_metadata: version invalid")),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Version two is reserved for permanent artifact backup retention. Older
|
||||
/// readers reject its header instead of skipping the protection field.
|
||||
pub(crate) fn decode_file(bytes: &[u8]) -> Result<Self> {
|
||||
Self::check_header(bytes)?;
|
||||
let metadata = Self::unmarshal(&bytes[4..])?;
|
||||
let retained_header = LittleEndian::read_u16(&bytes[2..4]) == 2;
|
||||
if retained_header != metadata.artifact_backup_retained {
|
||||
return Err(Error::other("artifact backup retention header mismatch"));
|
||||
}
|
||||
if metadata.artifact_backup_retained {
|
||||
metadata.validate_artifact_configuration()?;
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub(crate) fn default_timestamps(&mut self) {
|
||||
if self.policy_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.policy_config_updated_at = self.created
|
||||
@@ -1025,6 +1069,9 @@ impl BucketMetadata {
|
||||
}
|
||||
Some(config)
|
||||
};
|
||||
if self.artifact_base_prefix.is_some() && !config.as_ref().is_some_and(artifact_versioning_enabled) {
|
||||
return Err(Error::other("artifact bucket versioning must remain enabled"));
|
||||
}
|
||||
self.versioning_config_xml = data;
|
||||
self.versioning_config = config;
|
||||
self.versioning_config_updated_at = updated;
|
||||
@@ -1127,13 +1174,34 @@ impl BucketMetadata {
|
||||
&mut self,
|
||||
store: std::sync::Arc<crate::store::ECStore>,
|
||||
write_completion: WriteCompletion,
|
||||
) -> Result<()> {
|
||||
let opts = crate::object_api::ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion,
|
||||
..Default::default()
|
||||
};
|
||||
self.save_with_store_options(store, &opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_with_store_options(
|
||||
&mut self,
|
||||
store: Arc<ECStore>,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> Result<()> {
|
||||
self.parse_all_configs()?;
|
||||
self.validate_artifact_update(store.clone()).await?;
|
||||
let mut buf: Vec<u8> = vec![0; 4];
|
||||
|
||||
LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT);
|
||||
|
||||
LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION);
|
||||
LittleEndian::write_u16(
|
||||
&mut buf[2..4],
|
||||
if self.artifact_backup_retained {
|
||||
2
|
||||
} else {
|
||||
BUCKET_METADATA_VERSION
|
||||
},
|
||||
);
|
||||
|
||||
let data = self
|
||||
.marshal_msg()
|
||||
@@ -1141,21 +1209,52 @@ impl BucketMetadata {
|
||||
|
||||
buf.extend_from_slice(&data);
|
||||
|
||||
save_config_with_opts(
|
||||
store,
|
||||
self.save_file_path().as_str(),
|
||||
buf,
|
||||
&crate::object_api::ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
save_config_with_opts(store, self.save_file_path().as_str(), buf, opts).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_artifact_configuration(&self) -> Result<()> {
|
||||
if self.artifact_backup_retained && (self.artifact_base_prefix.is_none() || !self.bucket_creation_committed) {
|
||||
return Err(Error::other("artifact backup retention requires a committed managed bucket"));
|
||||
}
|
||||
if let Some(prefix) = &self.artifact_base_prefix {
|
||||
if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(prefix) || self.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other("invalid artifact bucket identity"));
|
||||
}
|
||||
let versioning = deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?;
|
||||
if !artifact_versioning_enabled(&versioning) {
|
||||
return Err(Error::other("artifact bucket versioning must remain enabled"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full metadata replacement must not change a generation's creation identity.
|
||||
pub(crate) async fn validate_artifact_update(&self, store: Arc<ECStore>) -> Result<()> {
|
||||
self.validate_artifact_configuration()?;
|
||||
// An existing empty/corrupt metadata object is not permission to create an identity.
|
||||
match read_config_preserve_empty(store, &self.save_file_path()).await {
|
||||
Ok(data) => {
|
||||
let previous = Self::decode_file(&data)?;
|
||||
previous.validate_artifact_configuration()?;
|
||||
if previous.artifact_backup_retained && !self.artifact_backup_retained {
|
||||
return Err(Error::other("artifact backup retention cannot be removed"));
|
||||
}
|
||||
// Deletion removes this record before recreation. A caller-supplied
|
||||
// incarnation cannot turn an existing bucket into a new generation.
|
||||
if previous.artifact_base_prefix != self.artifact_base_prefix
|
||||
|| (previous.artifact_base_prefix.is_some() && previous.bucket_incarnation_id != self.bucket_incarnation_id)
|
||||
{
|
||||
return Err(Error::other("artifact bucket identity is immutable"));
|
||||
}
|
||||
}
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_policy_config(&mut self) -> Result<()> {
|
||||
if !self.policy_config_json.is_empty() {
|
||||
self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?);
|
||||
@@ -1199,6 +1298,7 @@ impl BucketMetadata {
|
||||
/// `rustfs_bucket_metadata_parse_failed_total`, reflected in
|
||||
/// `rustfs_bucket_metadata_unparsable_current`, and logged at error level.
|
||||
pub(super) fn parse_all_configs(&mut self) -> Result<()> {
|
||||
self.validate_artifact_configuration()?;
|
||||
if let Err(e) = self.parse_policy_config() {
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1464,6 +1564,10 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
fn artifact_versioning_enabled(config: &VersioningConfiguration) -> bool {
|
||||
config.enabled() && !config.exclude_folders.unwrap_or(false) && config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_bucket_incarnation(api: Arc<ECStore>, bucket: &str) -> Result<Option<Uuid>> {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}");
|
||||
let data = match read_config_preserve_empty(api, &path).await {
|
||||
@@ -1560,9 +1664,7 @@ async fn read_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketM
|
||||
|
||||
let data = read_config(api, &file_path).await?;
|
||||
|
||||
BucketMetadata::check_header(&data)?;
|
||||
|
||||
let bm = BucketMetadata::unmarshal(&data[4..])?;
|
||||
let bm = BucketMetadata::decode_file(&data)?;
|
||||
|
||||
Ok(bm)
|
||||
}
|
||||
@@ -1586,6 +1688,98 @@ where
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn artifact_metadata_roundtrip_and_corrupt_prefix_rejection() {
|
||||
let mut metadata = BucketMetadata::new("artifacts");
|
||||
metadata.artifact_base_prefix = Some("images/".to_owned());
|
||||
let encoded = metadata.marshal_msg().expect("encode artifact metadata");
|
||||
let decoded = BucketMetadata::unmarshal(&encoded).expect("decode artifact metadata");
|
||||
assert_eq!(decoded.artifact_base_prefix, metadata.artifact_base_prefix, "ARTIFACT_METADATA_ROUNDTRIP");
|
||||
assert_eq!(decoded.bucket_incarnation_id, metadata.bucket_incarnation_id);
|
||||
let legacy = BucketMetadata::new("ordinary")
|
||||
.marshal_msg()
|
||||
.expect("encode ordinary metadata");
|
||||
assert!(
|
||||
BucketMetadata::unmarshal(&legacy)
|
||||
.expect("decode ordinary metadata")
|
||||
.artifact_base_prefix
|
||||
.is_none()
|
||||
);
|
||||
for prefix in ["", "../", "outputs", "a b/"] {
|
||||
let mut invalid = Vec::new();
|
||||
rmp::encode::write_map_len(&mut invalid, 1).expect("encode map");
|
||||
rmp::encode::write_str(&mut invalid, "ArtifactBasePrefix").expect("encode field");
|
||||
rmp::encode::write_str(&mut invalid, prefix).expect("encode invalid prefix");
|
||||
assert!(BucketMetadata::unmarshal(&invalid).is_err(), "ARTIFACT_CORRUPT_PREFIX");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_backup_retention_marker_rejects_false_and_duplicates() {
|
||||
for count in [1, 2] {
|
||||
let mut bytes = Vec::new();
|
||||
rmp::encode::write_map_len(&mut bytes, count).unwrap();
|
||||
for _ in 0..count {
|
||||
rmp::encode::write_str(&mut bytes, "ArtifactBackupRetained").unwrap();
|
||||
rmp::encode::write_bool(&mut bytes, count == 2).unwrap();
|
||||
}
|
||||
assert!(BucketMetadata::unmarshal(&bytes).is_err(), "O02P3_INVALID_RETENTION_MARKER");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_backup_retention_requires_matching_header_and_identity() {
|
||||
let mut metadata = BucketMetadata::new("retained");
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
metadata.bucket_creation_committed = true;
|
||||
metadata.artifact_base_prefix = Some("outputs/".to_owned());
|
||||
metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
|
||||
let file = |metadata: &BucketMetadata, version: u16| {
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(&BUCKET_METADATA_FORMAT.to_le_bytes());
|
||||
bytes.extend_from_slice(&version.to_le_bytes());
|
||||
bytes.extend_from_slice(&metadata.marshal_msg().unwrap());
|
||||
bytes
|
||||
};
|
||||
assert!(BucketMetadata::decode_file(&file(&metadata, 1)).is_ok(), "O02P3_ORDINARY_V1_COMPATIBLE");
|
||||
assert!(BucketMetadata::decode_file(&file(&metadata, 2)).is_err(), "O02P3_MARKER_REQUIRED");
|
||||
metadata.artifact_backup_retained = true;
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 1)).is_err(),
|
||||
"O02P3_PROTECTION_CANNOT_DOWNGRADE_HEADER"
|
||||
);
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 2))
|
||||
.unwrap()
|
||||
.artifact_backup_retained,
|
||||
"O02P3_PROTECTED_V2_ROUNDTRIP"
|
||||
);
|
||||
metadata.artifact_base_prefix = None;
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 2)).is_err(),
|
||||
"O02P3_PROTECTION_REQUIRES_MANAGED_BUCKET"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_versioning_cannot_be_disabled_or_excluded() {
|
||||
let mut metadata = BucketMetadata::new("artifacts");
|
||||
metadata.artifact_base_prefix = Some("images/".to_owned());
|
||||
for xml in [
|
||||
"",
|
||||
"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>",
|
||||
"<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>",
|
||||
"<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>images/</Prefix></ExcludedPrefixes></VersioningConfiguration>",
|
||||
] {
|
||||
assert!(
|
||||
metadata
|
||||
.update_config(BUCKET_VERSIONING_CONFIG, xml.as_bytes().to_vec())
|
||||
.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1734: `StorageError::clone` rebuilds I/O errors from
|
||||
/// their text. The unreadable-config refusal must stay typed across that
|
||||
/// clone, or a cloned error stops mapping to its retryable S3 response.
|
||||
|
||||
@@ -293,8 +293,10 @@ pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>
|
||||
pub async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let lock = sys.write().await;
|
||||
lock.set(bucket, Arc::new(bm)).await;
|
||||
Ok(())
|
||||
if bucket != bm.name {
|
||||
return Err(Error::other("bucket metadata identity mismatch"));
|
||||
}
|
||||
lock.set_validated(bm).await
|
||||
}
|
||||
|
||||
pub async fn reload_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
@@ -585,6 +587,15 @@ pub(crate) async fn get_cached_bucket_incarnation_id_in(
|
||||
sys.get_bucket_incarnation_id(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_incarnation_metadata_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
sys.get_bucket_incarnation_metadata(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn set_bucket_metadata_in(ctx: &crate::runtime::instance::InstanceContext, bm: BucketMetadata) -> Result<()> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let lock = sys.read().await;
|
||||
@@ -2132,7 +2143,16 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
pub(crate) async fn set_validated(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
if bm.artifact_base_prefix.is_some() {
|
||||
bm.parse_all_configs()?;
|
||||
}
|
||||
bm.validate_artifact_update(self.object_store()).await?;
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2147,6 +2167,11 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn set_for_test(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.set(bucket, bm).await;
|
||||
}
|
||||
|
||||
/// Remove a bucket's cached metadata from the in-memory map.
|
||||
///
|
||||
/// Returns `true` if an entry was present. Reserved meta buckets are ignored.
|
||||
@@ -2651,12 +2676,16 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
Ok(self.get_bucket_incarnation_metadata(bucket).await?.bucket_incarnation_id)
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let authority = self.get_metadata_authority(bucket).await?;
|
||||
if let BucketMetadataAuthority::Authoritative(metadata) = &authority
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& !metadata.bucket_incarnation_id.is_nil()
|
||||
{
|
||||
return Ok(metadata.bucket_incarnation_id);
|
||||
return Ok(Arc::clone(metadata));
|
||||
}
|
||||
|
||||
match authority {
|
||||
@@ -2665,7 +2694,7 @@ impl BucketMetadataSys {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
Ok(metadata.bucket_incarnation_id)
|
||||
Ok(metadata)
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) | BucketMetadataAuthority::Fabricated => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
@@ -2993,7 +3022,8 @@ impl BucketMetadataSys {
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
|
||||
if persisted && (metadata.lock_enabled || metadata.artifact_base_prefix.is_some()) && !metadata.bucket_creation_committed
|
||||
{
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if persisted {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||
|
||||
pub mod artifact;
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
pub mod config_parse_mode;
|
||||
|
||||
@@ -895,7 +895,7 @@ mod tests {
|
||||
.update_bucket_metadata_config(&bucket, QUOTA_CONFIG_FILE, legacy_quota_json(8192))
|
||||
.await
|
||||
.expect("newer quota save");
|
||||
sys.read().await.set(bucket.clone(), stale).await;
|
||||
sys.read().await.set_for_test(bucket.clone(), stale).await;
|
||||
assert_eq!(
|
||||
checker
|
||||
.migrate_reservation_protocol(&bucket)
|
||||
|
||||
@@ -275,7 +275,7 @@ async fn quota_statistics_remain_available_without_a_complete_scanner_snapshot()
|
||||
// assertion: reporting must work before a protocol upgrade.
|
||||
let metadata_sys = BucketMetadataSys::new(store.clone());
|
||||
let (metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
|
||||
metadata_sys.set(bucket.clone(), metadata).await;
|
||||
metadata_sys.set_for_test(bucket.clone(), metadata).await;
|
||||
let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
|
||||
let (snapshot_source, size) = if source == "empty" { ("complete", 0) } else { (source, 1000) };
|
||||
install_reporting_snapshot(store.clone(), &bucket, snapshot_source, size).await;
|
||||
@@ -317,7 +317,7 @@ async fn quota_statistics_preserve_configuration_errors() {
|
||||
let metadata_sys = BucketMetadataSys::new(store.clone());
|
||||
let (mut metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
|
||||
Arc::make_mut(&mut metadata).quota_config_json = b"{not-json".to_vec();
|
||||
metadata_sys.set(bucket.clone(), metadata).await;
|
||||
metadata_sys.set_for_test(bucket.clone(), metadata).await;
|
||||
let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
|
||||
|
||||
let statistics = checker.get_quota_stats(&bucket).await;
|
||||
|
||||
@@ -2375,7 +2375,300 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
file_info.validate_for_metadata_read().map_err(Into::into)
|
||||
}
|
||||
|
||||
fn validate_artifact_repair_seal_response(
|
||||
request: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest,
|
||||
response: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse,
|
||||
) -> Result<()> {
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
|
||||
.map_err(|_| Error::other("artifact repair seal response is too large"))?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
.map_err(|_| Error::other("artifact repair seal response proof is invalid"))?;
|
||||
if !response.success {
|
||||
return Err(response.error.clone().unwrap_or_default().into());
|
||||
}
|
||||
let expected_disk = Uuid::from_slice(&request.expected_disk_id).ok().filter(|id| !id.is_nil());
|
||||
if response.error.is_some() || expected_disk.is_none() || Uuid::from_slice(&response.disk_id).ok() != expected_disk {
|
||||
return Err(Error::other("artifact repair seal response disk identity mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) async fn rename_artifact_repair_data(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
expected_incarnation: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"artifact_repair_rename_data",
|
||||
|| async {
|
||||
let body = ArtifactRepairRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: source.0.to_owned(),
|
||||
src_path: source.1.to_owned(),
|
||||
dst_volume: destination.0.to_owned(),
|
||||
dst_path: destination.1.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair rename request is too large"))?;
|
||||
let mut request = Request::new(body);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
// Unimplemented is terminal: never downgrade repair publication to PUT or legacy RPCs.
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.artifact_repair_rename_data(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn write_artifact_repair_metadata(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
target: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
expected_incarnation: Uuid,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"artifact_repair_write_metadata",
|
||||
|| async {
|
||||
let body = ArtifactRepairWriteMetadataRequest {
|
||||
metadata: Some(WriteMetadataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: target.0.to_owned(),
|
||||
path: target.1.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_write_metadata_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair metadata request is too large"))?;
|
||||
let mut request = Request::new(body);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.artifact_repair_write_metadata(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// One authenticated scan (or explicitly distinct read-only identity probe).
|
||||
/// Partial observations are never retried after output has begun.
|
||||
pub async fn inventory_artifact_attempt(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
target: (&str, Uuid),
|
||||
prefix: &str,
|
||||
phase: rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase,
|
||||
deadline: tokio::time::Instant,
|
||||
output: tokio::sync::mpsc::Sender<crate::disk::ArtifactInventoryFrame>,
|
||||
) -> Result<()> {
|
||||
use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
use rustfs_protos::{
|
||||
artifact_inventory as protocol,
|
||||
proto_gen::node_service::{ArtifactInventoryRequest, artifact_inventory_frame::Payload},
|
||||
};
|
||||
let (bucket, incarnation) = target;
|
||||
let duration = deadline
|
||||
.checked_duration_since(tokio::time::Instant::now())
|
||||
.ok_or(Error::Timeout)?;
|
||||
let body = ArtifactInventoryRequest {
|
||||
protocol_version: protocol::VERSION,
|
||||
invocation_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
disk: self.endpoint.to_string(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: prefix.to_owned(),
|
||||
timeout_millis: u64::try_from(duration.as_millis().min(u128::from(protocol::MAX_TIMEOUT_MILLIS)))
|
||||
.map_err(Error::other)?,
|
||||
phase: phase.into(),
|
||||
};
|
||||
protocol::validate_request(&body).map_err(Error::other)?;
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
let canonical = protocol::canonical_request(&body).map_err(Error::other)?;
|
||||
let mut request = Request::new(body.clone());
|
||||
request.set_timeout(duration);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let mut stream = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(protocol::MAX_FRAME_BYTES)
|
||||
.inventory_artifact_attempt(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
let mut state = protocol::StreamState::new();
|
||||
let mut terminal = None;
|
||||
while let Some(frame) = stream.message().await? {
|
||||
state
|
||||
.accept(&body, &frame, |canonical, proof| {
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(canonical, proof).is_ok()
|
||||
})
|
||||
.map_err(Error::other)?;
|
||||
match frame.payload {
|
||||
Some(Payload::Entries(entries)) => {
|
||||
let versions = entries
|
||||
.versions
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
Ok(ArtifactInventoryVersion {
|
||||
key: entry.key,
|
||||
version_id: Uuid::from_slice(&entry.version_id).map_err(Error::other)?,
|
||||
delete_marker: entry.delete_marker,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
output
|
||||
.send(ArtifactInventoryFrame::Entries {
|
||||
sequence: frame.sequence,
|
||||
versions,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
}
|
||||
Some(Payload::Complete(complete)) => terminal = Some(complete.entries),
|
||||
Some(Payload::Verified(_)) => {}
|
||||
None => return Err(Error::other("artifact inventory payload is absent")),
|
||||
}
|
||||
}
|
||||
// A valid Complete followed by a transport failure or extra frame is
|
||||
// still incomplete. Publish the local terminal only after clean EOF.
|
||||
state.finish().map_err(Error::other)?;
|
||||
if let Some(entries) = terminal {
|
||||
output
|
||||
.send(ArtifactInventoryFrame::Complete {
|
||||
disk_id: expected_disk,
|
||||
incarnation,
|
||||
attempt_prefix: prefix.to_owned(),
|
||||
entries,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::Timeout)?
|
||||
}
|
||||
|
||||
pub(crate) async fn seal_artifact_puts(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"seal_artifact_puts",
|
||||
|| async {
|
||||
let body = SealArtifactAttemptPutsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: attempt_prefix.to_owned(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact seal request is too large"))?;
|
||||
let mut request = Request::new(body.clone());
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.seal_artifact_attempt_puts(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&body, &response)
|
||||
.map_err(|_| Error::other("artifact seal response is too large"))?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
.map_err(|_| Error::other("artifact seal response proof is invalid"))?;
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.error.is_some() || Uuid::from_slice(&response.disk_id).ok() != Some(expected_disk) {
|
||||
return Err(Error::other("artifact seal response disk identity mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn seal_artifact_repairs(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
|
||||
if expected_disk.is_nil() || incarnation.is_nil() {
|
||||
return Err(Error::other("artifact repair seal requires non-nil disk and incarnation identities"));
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"seal_artifact_repairs",
|
||||
|| async {
|
||||
let body = SealArtifactAttemptPutsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: attempt_prefix.to_owned(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair seal request is too large"))?;
|
||||
let mut request = Request::new(body.clone());
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.seal_artifact_attempt_repairs(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
validate_artifact_repair_seal_response(&body, &response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -2385,20 +2678,39 @@ impl RemoteDisk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
self.rename_data_borrowed_with_fence((src_volume, src_path), fi, (dst_volume, dst_path), None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
target: (&str, &str),
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
artifact_incarnation: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
let (src_volume, src_path) = source;
|
||||
let (dst_volume, dst_path) = target;
|
||||
if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path)
|
||||
&& let Some(identity) = scope.receiver_identity()
|
||||
{
|
||||
scope.check().map_err(Error::other)?;
|
||||
if artifact_incarnation.is_some() || scanner_publication_lease_token.is_some() {
|
||||
return Err(Error::other("artifact repair cannot use a publication lease"));
|
||||
}
|
||||
let expected = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
|
||||
let result = self
|
||||
.rename_artifact_repair_data(expected, (src_volume, src_path), fi, (dst_volume, dst_path), identity.incarnation)
|
||||
.await;
|
||||
scope.check().map_err(Error::other)?;
|
||||
return result;
|
||||
}
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2421,8 +2733,11 @@ impl RemoteDisk {
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
|
||||
.map(|scope| scope.incarnation.as_bytes().to_vec().into())
|
||||
bucket_incarnation_id: artifact_incarnation
|
||||
.or_else(|| {
|
||||
crate::store::bucket_heal_scope_for_object(dst_volume, dst_path).map(|scope| scope.incarnation)
|
||||
})
|
||||
.map(|incarnation| incarnation.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
@@ -2435,7 +2750,11 @@ impl RemoteDisk {
|
||||
.map(|token| token.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
let canonical_body = if artifact_incarnation.is_some() {
|
||||
rustfs_protos::canonical_artifact_rename_data_request_body(request.get_ref())
|
||||
} else {
|
||||
rustfs_protos::canonical_rename_data_request_body(request.get_ref())
|
||||
};
|
||||
let incarnation_bound = !request.get_ref().bucket_incarnation_id.is_empty();
|
||||
let quota_fenced = rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
@@ -2458,7 +2777,9 @@ impl RemoteDisk {
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if quota_fenced {
|
||||
if artifact_incarnation.is_some() {
|
||||
client.artifact_rename_data(request).await
|
||||
} else if quota_fenced {
|
||||
// The method itself proves quota-token enforcement; an old peer must fail closed.
|
||||
client.rename_data_with_quota_fence(request).await
|
||||
} else if incarnation_bound {
|
||||
@@ -3204,6 +3525,21 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(volume, path)
|
||||
&& let Some(identity) = scope.receiver_identity()
|
||||
{
|
||||
scope.check().map_err(Error::other)?;
|
||||
let expected = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
|
||||
let result = self
|
||||
.write_artifact_repair_metadata(expected, (volume, path), &fi, identity.incarnation)
|
||||
.await;
|
||||
scope.check().map_err(Error::other)?;
|
||||
return result;
|
||||
}
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -4553,6 +4889,83 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_old_peer_unimplemented_never_downgrades_mutations() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("loopback test peer");
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse(&format!("{}/data/rustfs0", peer.addr)).expect("test disk endpoint"),
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: 0,
|
||||
};
|
||||
let disk = RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("remote test disk");
|
||||
let expected_disk = Uuid::new_v4();
|
||||
let incarnation = Uuid::new_v4();
|
||||
let info = FileInfo::default();
|
||||
let rename = disk
|
||||
.rename_artifact_repair_data(
|
||||
expected_disk,
|
||||
("tmp", "staged"),
|
||||
&info,
|
||||
("artifacts", "outputs/tenant/run/attempt/image"),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot publish repair data");
|
||||
let metadata = disk
|
||||
.write_artifact_repair_metadata(expected_disk, ("artifacts", "outputs/tenant/run/attempt/image"), &info, incarnation)
|
||||
.await
|
||||
.expect_err("old peer cannot publish repair metadata");
|
||||
let seal = disk
|
||||
.seal_artifact_repairs(expected_disk, "artifacts", incarnation, "outputs/tenant/run/attempt")
|
||||
.await
|
||||
.expect_err("old peer cannot confirm repair closure");
|
||||
for error in [rename, metadata, seal] {
|
||||
let DiskError::Io(error) = error else {
|
||||
panic!("expected gRPC status error");
|
||||
};
|
||||
let status = error
|
||||
.get_ref()
|
||||
.and_then(|error| error.downcast_ref::<crate::disk::error::RpcStatusError>())
|
||||
.expect("preserved gRPC status");
|
||||
assert_eq!(status.status().code(), Code::Unimplemented);
|
||||
}
|
||||
let paths = peer.peer.rpc_paths.lock().expect("RPC inventory").clone();
|
||||
for method in [
|
||||
"ArtifactRepairRenameData",
|
||||
"ArtifactRepairWriteMetadata",
|
||||
"SealArtifactAttemptRepairs",
|
||||
] {
|
||||
assert_eq!(
|
||||
paths.iter().filter(|path| path.ends_with(&format!("/{method}"))).count(),
|
||||
1,
|
||||
"each repair method is sent once"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
paths.iter().all(|path| matches!(
|
||||
path.rsplit('/').next(),
|
||||
Some("ArtifactRepairRenameData" | "ArtifactRepairWriteMetadata" | "SealArtifactAttemptRepairs" | "Ping")
|
||||
)),
|
||||
"repair cannot downgrade to a legacy mutation: {paths:?}"
|
||||
);
|
||||
disk.cancel_token.cancel();
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -4648,6 +5061,93 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_repair_seal_response_requires_proof_full_request_and_physical_identity() {
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
Error as WireError, SealArtifactAttemptPutsRequest, SealArtifactAttemptPutsResponse,
|
||||
};
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let request = SealArtifactAttemptPutsRequest {
|
||||
disk: "disk-a".into(),
|
||||
bucket: "artifacts".into(),
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
attempt_prefix: "outputs/tenant/run/attempt".into(),
|
||||
expected_disk_id: vec![2; 16].into(),
|
||||
};
|
||||
let mut response = SealArtifactAttemptPutsResponse {
|
||||
success: true,
|
||||
disk_id: request.expected_disk_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let sign = |request: &SealArtifactAttemptPutsRequest, response: &mut SealArtifactAttemptPutsResponse| {
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
|
||||
.expect("repair response canonical");
|
||||
response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&canonical)
|
||||
.expect("repair response proof")
|
||||
.into();
|
||||
};
|
||||
assert!(validate_artifact_repair_seal_response(&request, &response).is_err(), "proof is mandatory");
|
||||
sign(&request, &mut response);
|
||||
validate_artifact_repair_seal_response(&request, &response).expect("signed exact physical receiver");
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.disk = "other-disk".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket = "other-bucket".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = vec![3; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.attempt_prefix = "other-attempt".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = vec![4; 16].into(),
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&changed, &response).is_err(),
|
||||
"another request cannot reuse the proof"
|
||||
);
|
||||
}
|
||||
let legacy = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&request, &response).expect("PUT domain");
|
||||
let mut legacy_response = response.clone();
|
||||
legacy_response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&legacy)
|
||||
.expect("PUT proof")
|
||||
.into();
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &legacy_response).is_err(),
|
||||
"PUT proof cannot acknowledge repair seal"
|
||||
);
|
||||
for disk in [vec![], vec![0; 16], vec![3; 16], vec![1; 15]] {
|
||||
let mut changed = response.clone();
|
||||
changed.disk_id = disk.into();
|
||||
sign(&request, &mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"signed wrong or absent physical identity is invalid"
|
||||
);
|
||||
}
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.success = false,
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.error = Some(WireError::default()),
|
||||
] {
|
||||
let mut changed = response.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"response fields are bound"
|
||||
);
|
||||
sign(&request, &mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"signed failure or inconsistent success is invalid"
|
||||
);
|
||||
}
|
||||
let mut nil_request = request;
|
||||
nil_request.expected_disk_id = vec![0; 16].into();
|
||||
let mut nil_response = response;
|
||||
nil_response.disk_id = nil_request.expected_disk_id.clone();
|
||||
sign(&nil_request, &mut nil_response);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&nil_request, &nil_response).is_err(),
|
||||
"nil physical identity is never a receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() {
|
||||
let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2)
|
||||
@@ -5038,6 +5538,7 @@ mod tests {
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
rpc_paths: Arc<StdMutex<Vec<String>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
@@ -5057,6 +5558,7 @@ mod tests {
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
rpc_paths: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -5112,6 +5614,11 @@ mod tests {
|
||||
}
|
||||
|
||||
fn call(&mut self, request: http::Request<B>) -> Self::Future {
|
||||
self.peer
|
||||
.rpc_paths
|
||||
.lock()
|
||||
.expect("RPC path inventory")
|
||||
.push(request.uri().path().to_owned());
|
||||
match request.uri().path() {
|
||||
"/node_service.NodeService/DiskInfo" => {
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -438,6 +438,64 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: false,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation_no_lock<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if !guard.matches(RUSTFS_META_BUCKET, file) {
|
||||
return Err(Error::InvalidArgument(
|
||||
RUSTFS_META_BUCKET.to_owned(),
|
||||
file.to_owned(),
|
||||
"locked observation requires a matching write owner".to_owned(),
|
||||
));
|
||||
}
|
||||
let lock_lost_error = || Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
bucket: RUSTFS_META_BUCKET.to_owned(),
|
||||
object: file.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: true,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
let result = crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -484,10 +542,22 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
|
||||
let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Preserve an empty config payload, its metadata, and caller-supplied read options.
|
||||
pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, None).await
|
||||
}
|
||||
|
||||
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -516,15 +586,13 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(
|
||||
read_config_preserve_empty_with_metadata_opts(
|
||||
api,
|
||||
file,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2919,9 +2987,9 @@ mod tests {
|
||||
config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
|
||||
heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
|
||||
new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
|
||||
read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
|
||||
save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_transaction_lock_path, storage_class_kvs_mut,
|
||||
read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
|
||||
read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, heal, notify, oidc, scanner};
|
||||
use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||
@@ -5396,6 +5464,7 @@ mod tests {
|
||||
heal_replacement: Option<Vec<u8>>,
|
||||
heal_calls: AtomicUsize,
|
||||
write_calls: AtomicUsize,
|
||||
last_get_no_lock: AtomicBool,
|
||||
last_put_no_lock: AtomicBool,
|
||||
last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
|
||||
revision: AtomicUsize,
|
||||
@@ -5411,6 +5480,7 @@ mod tests {
|
||||
heal_replacement,
|
||||
heal_calls: AtomicUsize::new(0),
|
||||
write_calls: AtomicUsize::new(0),
|
||||
last_get_no_lock: AtomicBool::new(false),
|
||||
last_put_no_lock: AtomicBool::new(false),
|
||||
last_put_preconditions: Mutex::new(None),
|
||||
revision: AtomicUsize::new(1),
|
||||
@@ -5466,8 +5536,9 @@ mod tests {
|
||||
_object: &str,
|
||||
_range: Option<HTTPRangeSpec>,
|
||||
_h: HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
|
||||
let data = match &*self.state.lock().expect("state lock poisoned") {
|
||||
RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
|
||||
RecoveryReadState::Blob(data) => data.clone(),
|
||||
@@ -5779,6 +5850,325 @@ mod tests {
|
||||
assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
type ObservationEofGate = (Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
struct ObservationStream {
|
||||
inner: Cursor<Vec<u8>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
fail: bool,
|
||||
eof_gate: Option<ObservationEofGate>,
|
||||
}
|
||||
|
||||
impl AsyncRead for ObservationStream {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.fail {
|
||||
return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
|
||||
}
|
||||
if self.inner.position() == u64::try_from(self.inner.get_ref().len()).expect("fixture length")
|
||||
&& let Some((ready, release)) = self.eof_gate.as_mut()
|
||||
{
|
||||
if let Some(ready) = ready.take() {
|
||||
ready.send(()).expect("EOF observer remains active");
|
||||
}
|
||||
match std::future::Future::poll(Pin::new(release), cx) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(result) => result.expect("EOF gate released"),
|
||||
}
|
||||
self.eof_gate = None;
|
||||
}
|
||||
let before = buf.filled().len();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationStore {
|
||||
body: Vec<u8>,
|
||||
declared_size: i64,
|
||||
error: Option<Error>,
|
||||
stream_error: bool,
|
||||
gets: AtomicUsize,
|
||||
eof_gate: std::sync::Mutex<Option<ObservationEofGate>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
options: std::sync::Mutex<Option<(bool, bool, bool)>>,
|
||||
writes: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Debug for ObservationStore {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ObservationStore {
|
||||
fn new(body: Vec<u8>, declared_size: i64) -> Self {
|
||||
Self {
|
||||
body,
|
||||
declared_size,
|
||||
error: None,
|
||||
stream_error: false,
|
||||
gets: AtomicUsize::new(0),
|
||||
eof_gate: std::sync::Mutex::new(None),
|
||||
consumed: Arc::new(AtomicUsize::new(0)),
|
||||
options: std::sync::Mutex::new(None),
|
||||
writes: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
_headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
assert_eq!(bucket, RUSTFS_META_BUCKET);
|
||||
assert_eq!(object, "config/observation.bin");
|
||||
self.gets.fetch_add(1, Ordering::SeqCst);
|
||||
assert!(range.is_none());
|
||||
*self.options.lock().expect("observation options") =
|
||||
Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
|
||||
if let Some(error) = &self.error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
Ok(GetObjectReader {
|
||||
stream: Box::new(ObservationStream {
|
||||
inner: Cursor::new(self.body.clone()),
|
||||
consumed: self.consumed.clone(),
|
||||
fail: self.stream_error,
|
||||
eof_gate: self.eof_gate.lock().expect("EOF gate").take(),
|
||||
}),
|
||||
object_info: ObjectInfo {
|
||||
size: self.declared_size,
|
||||
etag: Some("observation-etag".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.writes.fetch_add(1, Ordering::SeqCst);
|
||||
Err(Error::other("observation must not write"))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_preserves_options_and_typed_errors() {
|
||||
let object = "config/observation.bin";
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-options".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-options".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert!(result.is_ok(), "I06L_READ_FLAGS: {result:?}");
|
||||
assert!(*store.options.lock().expect("options") == Some((true, true, false)), "I06L_READ_FLAGS");
|
||||
assert_eq!(result.expect("raw observation").0, b"raw");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
for (bucket, key) in [(RUSTFS_META_BUCKET, "config/other.bin"), ("wrong-bucket", object)] {
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-target".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(bucket, key),
|
||||
"locked-observation-target".to_owned(),
|
||||
);
|
||||
let wrong = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("wrong-target real owner");
|
||||
let store = Arc::new(ObservationStore::new(Vec::new(), 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &wrong).await;
|
||||
assert!(matches!(result, Err(Error::InvalidArgument(..))), "I06L_TARGET_REJECTED");
|
||||
assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_TARGET_REJECTED");
|
||||
}
|
||||
for error in [
|
||||
Error::FileNotFound,
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_owned(), object.to_owned()),
|
||||
Error::ErasureReadQuorum,
|
||||
] {
|
||||
let expected = if error == Error::FileNotFound || matches!(error, Error::ObjectNotFound(..)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
error.clone()
|
||||
};
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(error);
|
||||
let store = Arc::new(fixture);
|
||||
assert_eq!(
|
||||
super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("typed error"),
|
||||
expected
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let err = super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("stream failure");
|
||||
assert!(err.to_string().contains("observation stream failed"), "I06L_TYPED_STREAM_ERROR");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &guard).await;
|
||||
assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_STREAM_BOUND");
|
||||
assert!(store.consumed.load(Ordering::SeqCst) == 9, "I06L_STREAM_BOUND");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_rejects_actual_owner_loss_before_and_after_read() {
|
||||
let object = "config/observation.bin";
|
||||
for after_read in [false, true] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let client = Arc::new(RefreshControlledClient::new(manager));
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::new("locked-observation-loss".to_owned(), client.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-loss".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("real distributed owner");
|
||||
let signal = guard.lock_lost_signal().expect("acquired distributed owner signal");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
if after_read {
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
*store.eof_gate.lock().expect("EOF gate") = Some((Some(ready_tx), release_rx));
|
||||
let mut read = Box::pin(super::read_config_observation_no_lock(store.clone(), object, 3, &guard));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
tokio::select! {
|
||||
ready = ready_rx => ready.expect("read reached EOF"),
|
||||
result = &mut read => assert_eq!(false, true, "I06L_LOSS_AFTER_READ: returned before EOF gate: {result:?}"),
|
||||
}
|
||||
}).await.expect("EOF watchdog");
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert!(guard.is_lock_lost(), "I06L_LOSS_AFTER_READ");
|
||||
release_tx.send(()).expect("release EOF");
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(60), read)
|
||||
.await
|
||||
.expect("read watchdog");
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
"I06L_LOSS_AFTER_READ"
|
||||
);
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst), 1);
|
||||
} else {
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert!(guard.is_lock_lost(), "I06L_LOSS_BEFORE_GET");
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
"I06L_LOSS_BEFORE_GET"
|
||||
);
|
||||
assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_LOSS_BEFORE_GET");
|
||||
}
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_preserves_options_and_errors() {
|
||||
for body in [Vec::new(), b"raw config".to_vec()] {
|
||||
let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
|
||||
let (observed, info) = result.expect("successful observation");
|
||||
assert_eq!(observed, body);
|
||||
assert_eq!(info.etag.as_deref(), Some("observation-etag"));
|
||||
assert_eq!(
|
||||
*store.options.lock().expect("options"),
|
||||
Some((true, false, false)),
|
||||
"I05_REPAIR_SUPPRESSED"
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
for (source, expected) in [
|
||||
(Error::FileNotFound, Error::ConfigNotFound),
|
||||
(
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
|
||||
Error::ConfigNotFound,
|
||||
),
|
||||
(Error::ErasureReadQuorum, Error::ErasureReadQuorum),
|
||||
] {
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(source);
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert_eq!(result.expect_err("read error must propagate"), expected);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
let error = result.expect_err("stream error must propagate");
|
||||
assert!(error.to_string().contains("observation stream failed"));
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_enforces_byte_limit() {
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert!(result.is_err(), "I05_BOUND_ENFORCED");
|
||||
assert_eq!(
|
||||
result.expect_err("bounded observation error"),
|
||||
Error::CorruptedFormat,
|
||||
"I05_BOUND_ENFORCED"
|
||||
);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
|
||||
assert_eq!(result.expect("exact limit").0, vec![7; 8]);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_distinguishes_empty_object() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
@@ -5799,6 +6189,60 @@ mod tests {
|
||||
assert!(data.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
for no_lock in [false, true] {
|
||||
let opts = ObjectOptions {
|
||||
no_lock,
|
||||
..Default::default()
|
||||
};
|
||||
let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
|
||||
.await
|
||||
.expect("an existing empty config must retain its payload and metadata");
|
||||
assert!(data.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
assert_eq!(info.actual_size, 0);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
|
||||
}
|
||||
let err = read_config(store, "config/empty.json")
|
||||
.await
|
||||
.expect_err("the existing reader must still treat empty configs as missing");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
|
||||
let payload = b"original config bytes".to_vec();
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
|
||||
let (data, info) =
|
||||
read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("a nonempty config must retain its payload and metadata");
|
||||
assert_eq!(data, payload);
|
||||
let size = i64::try_from(payload.len()).expect("test payload size fits i64");
|
||||
assert_eq!(info.size, size);
|
||||
assert_eq!(info.actual_size, size);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_preserves_read_errors() {
|
||||
let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("physical absence must remain ConfigNotFound");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a quorum failure must retain its typed read error");
|
||||
assert!(matches!(err, Error::ErasureReadQuorum));
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl StorageAdminApi for RecoveryMockStore {
|
||||
type BackendInfo = rustfs_madmin::BackendInfo;
|
||||
|
||||
@@ -17076,6 +17076,7 @@ impl ECStore {
|
||||
&[],
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: None,
|
||||
lifecycle_owner: None,
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
|
||||
object_mutation_fence: Some(&object_fence),
|
||||
@@ -17344,7 +17345,7 @@ impl ECStore {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
|
||||
let bucket_incarnation_fence = match expected_bucket_incarnation_id {
|
||||
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||
Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
|
||||
None => None,
|
||||
};
|
||||
|
||||
@@ -18280,9 +18281,8 @@ impl ECStore {
|
||||
&cleanup_transferred_tier_versions,
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: expected_bucket_incarnation_id,
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
lifecycle_owner: bucket_incarnation_fence.as_ref(),
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: None,
|
||||
object_mutation_fence: Some(&source_cleanup_mutation_fence),
|
||||
},
|
||||
@@ -29408,6 +29408,7 @@ mod pools_tests {
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -499,6 +499,7 @@ mod decommission_lock_order_tests {
|
||||
),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
(store, refresh_calls)
|
||||
|
||||
@@ -99,6 +99,33 @@ pub struct Sets {
|
||||
ctx: Arc<InstanceContext>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Sets {
|
||||
pub(crate) fn from_test_set(
|
||||
set: Arc<SetDisks>,
|
||||
endpoints: PoolEndpoints,
|
||||
format: FormatV3,
|
||||
pool_idx: usize,
|
||||
parity_count: usize,
|
||||
ctx: Arc<InstanceContext>,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
id: format.id,
|
||||
disk_set: vec![set],
|
||||
pool_idx,
|
||||
set_count: 1,
|
||||
set_drive_count: endpoints.drives_per_set,
|
||||
endpoints,
|
||||
format,
|
||||
parity_count,
|
||||
default_parity_count: parity_count,
|
||||
distribution_algo: DistributionAlgoVersion::V1,
|
||||
exit_signal: None,
|
||||
ctx,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Sets {
|
||||
fn drop(&mut self) {
|
||||
if let Some(exit_signal) = self.exit_signal.take() {
|
||||
@@ -800,6 +827,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
bucket_incarnation_fence: dst_opts.bucket_incarnation_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ use crate::storage_api_contracts::{
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
|
||||
use crate::store::{BucketIncarnationFenceGuard, DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
@@ -1278,6 +1278,7 @@ pub(crate) enum SourceCleanupError {
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct SourceCleanupBucketFence<'a> {
|
||||
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
|
||||
pub(crate) lifecycle_owner: Option<&'a Arc<BucketIncarnationFenceGuard>>,
|
||||
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
pub(crate) namespace_lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
|
||||
@@ -1486,9 +1487,10 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
)
|
||||
};
|
||||
|
||||
if bucket_fence
|
||||
.lifecycle_guard
|
||||
.is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
|
||||
if bucket_fence.lifecycle_owner.is_some_and(|owner| owner.is_lock_lost())
|
||||
|| bucket_fence
|
||||
.lifecycle_guard
|
||||
.is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
|
||||
{
|
||||
return Err(SourceCleanupError::Storage(Error::other(format!(
|
||||
"{op_label}: bucket incarnation fence was lost before source cleanup"
|
||||
@@ -1527,6 +1529,9 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
|
||||
object_mutation_fence.add_namespace_lock_fence(&mut opts);
|
||||
}
|
||||
if let Some(owner) = bucket_fence.lifecycle_owner {
|
||||
owner.attach_owner(&mut opts);
|
||||
}
|
||||
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
|
||||
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
|
||||
}
|
||||
|
||||
@@ -3360,6 +3360,7 @@ mod tests {
|
||||
pool_meta_save_gate: TokioMutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -159,10 +159,6 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
std::fs::symlink_metadata(path)
|
||||
}
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir_all(path.as_ref()).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
e.raw_os_error() == Some(libc::EISDIR)
|
||||
|| e.kind() == io::ErrorKind::IsADirectory
|
||||
@@ -210,10 +206,6 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir(path.as_ref()).await
|
||||
}
|
||||
|
||||
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
@@ -472,17 +464,6 @@ mod tests {
|
||||
assert!(metadata.file_type().is_symlink());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_make_dir_all() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let nested_path = temp_dir.path().join("level1").join("level2").join("level3");
|
||||
|
||||
// Should create nested directories
|
||||
assert!(make_dir_all(&nested_path).await.is_ok());
|
||||
assert!(nested_path.exists());
|
||||
assert!(nested_path.is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remove_file() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
@@ -555,17 +536,6 @@ mod tests {
|
||||
assert!(!dir_path.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mkdir() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let dir_path = temp_dir.path().join("test_mkdir");
|
||||
|
||||
// Create directory
|
||||
assert!(mkdir(&dir_path).await.is_ok());
|
||||
assert!(dir_path.exists());
|
||||
assert!(dir_path.is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rename() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
+10203
-1074
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,740 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::error::{DiskError, Result};
|
||||
use std::{fs::File, sync::Arc};
|
||||
|
||||
/// A target disk's admitted artifact publication. Physical mutation workers
|
||||
/// retain this Arc until their final syscall and rollback have finished.
|
||||
pub struct ArtifactLease {
|
||||
_active: File,
|
||||
_key_directory: File,
|
||||
_seal_directory: File,
|
||||
_root: File,
|
||||
}
|
||||
|
||||
pub type ArtifactPutLease = ArtifactLease;
|
||||
pub type ArtifactRepairLease = ArtifactLease;
|
||||
|
||||
/// Read-only census owns both already-closed physical mutation domains.
|
||||
pub(super) struct ArtifactInventoryLease {
|
||||
_put: Arc<ArtifactLease>,
|
||||
_repair: Arc<ArtifactLease>,
|
||||
}
|
||||
|
||||
pub(super) async fn drain_closed(root: File, key: [u8; 32], deadline: tokio::time::Instant) -> Result<ArtifactInventoryLease> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::drain_closed(root, key, deadline).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, deadline);
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum Domain {
|
||||
Put,
|
||||
Repair,
|
||||
}
|
||||
|
||||
impl Domain {
|
||||
fn directory(self) -> &'static str {
|
||||
match self {
|
||||
Self::Put => ".rustfs-artifact-put-seals",
|
||||
Self::Repair => ".rustfs-artifact-repair-seals",
|
||||
}
|
||||
}
|
||||
|
||||
fn closed(self) -> &'static [u8] {
|
||||
match self {
|
||||
Self::Put => b"rustfs-artifact-put-sealed-v1\n",
|
||||
Self::Repair => b"rustfs-artifact-repair-sealed-v1\n",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pinned_root(disk: &super::LocalDisk) -> Result<File> {
|
||||
#[cfg(target_os = "linux")]
|
||||
let root = disk.mount_lease.clone();
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
let root = disk.artifact_identity_root.clone();
|
||||
#[cfg(not(unix))]
|
||||
let _ = disk;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
root.try_clone().map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
|
||||
pub(super) async fn acquire(root: File, key: [u8; 32]) -> Result<Arc<ArtifactPutLease>> {
|
||||
acquire_in(root, key, Domain::Put).await
|
||||
}
|
||||
|
||||
pub(super) async fn acquire_in(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::acquire(root, key, domain).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, domain);
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn seal(root: File, key: [u8; 32]) -> Result<()> {
|
||||
seal_in(root, key, Domain::Put).await
|
||||
}
|
||||
|
||||
pub(super) async fn seal_in(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::seal(root, key, domain).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, domain);
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
mod unix {
|
||||
use super::*;
|
||||
use rustix::fs::{FlockOperation, Mode, OFlags, flock, mkdirat, openat};
|
||||
use rustix::io::Errno;
|
||||
use std::{io, io::Read, io::Write, os::unix::fs::MetadataExt, time::Duration};
|
||||
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(10);
|
||||
const LOCK_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct Opened {
|
||||
root: File,
|
||||
seals: File,
|
||||
key: File,
|
||||
admission: File,
|
||||
active: File,
|
||||
domain: Domain,
|
||||
}
|
||||
|
||||
fn invalid() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "invalid artifact seal filesystem state")
|
||||
}
|
||||
|
||||
fn directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
match mkdirat(parent, name, Mode::RWXU) {
|
||||
Ok(()) | Err(Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_dir() || metadata.dev() != device || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn regular(file: &File, device: u64) -> io::Result<std::fs::Metadata> {
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() || metadata.dev() != device || metadata.nlink() != 1 || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
fn lock_file(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::CREATE | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)?);
|
||||
if regular(&file, device)?.len() != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
impl Opened {
|
||||
fn open(root: File, key: [u8; 32], domain: Domain) -> io::Result<Self> {
|
||||
let metadata = root.metadata()?;
|
||||
if !metadata.is_dir() {
|
||||
return Err(invalid());
|
||||
}
|
||||
let seals = directory(&root, domain.directory(), metadata.dev())?;
|
||||
let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
|
||||
let key = directory(&seals, &name, metadata.dev())?;
|
||||
// Neither lock inode is ever removed or replaced, including after
|
||||
// closing. Independent processes must always lock the same inode.
|
||||
let admission = lock_file(&key, "admission.lock", metadata.dev())?;
|
||||
let active = lock_file(&key, "active.lock", metadata.dev())?;
|
||||
Ok(Self {
|
||||
root,
|
||||
seals,
|
||||
key,
|
||||
admission,
|
||||
active,
|
||||
domain,
|
||||
})
|
||||
}
|
||||
|
||||
fn closed(&self) -> io::Result<Option<File>> {
|
||||
self.closed_with_flags(OFlags::RDWR)
|
||||
}
|
||||
|
||||
fn closed_with_flags(&self, access: OFlags) -> io::Result<Option<File>> {
|
||||
let mut file = match openat(
|
||||
&self.key,
|
||||
"closed",
|
||||
access | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::empty(),
|
||||
) {
|
||||
Ok(file) => File::from(file),
|
||||
Err(Errno::NOENT) => return Ok(None),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let length = u64::try_from(self.domain.closed().len()).map_err(|_| invalid())?;
|
||||
if regular(&file, self.root.metadata()?.dev())?.len() != length {
|
||||
return Err(invalid());
|
||||
}
|
||||
let mut bytes = vec![0; self.domain.closed().len()];
|
||||
file.read_exact(&mut bytes)?;
|
||||
if bytes != self.domain.closed() {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(Some(file))
|
||||
}
|
||||
|
||||
fn admit(self) -> io::Result<Arc<ArtifactLease>> {
|
||||
if self.closed()?.is_some() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::PermissionDenied,
|
||||
"artifact attempt mutation domain is closed",
|
||||
));
|
||||
}
|
||||
// Admission serializes the closed check with taking this shared
|
||||
// physical owner. A sealer cannot pass between those two steps.
|
||||
flock(&self.active, FlockOperation::NonBlockingLockShared)?;
|
||||
flock(&self.admission, FlockOperation::Unlock)?;
|
||||
Ok(Arc::new(ArtifactLease {
|
||||
_active: self.active,
|
||||
_key_directory: self.key,
|
||||
_seal_directory: self.seals,
|
||||
_root: self.root,
|
||||
}))
|
||||
}
|
||||
|
||||
fn close(self) -> io::Result<Self> {
|
||||
let marker = match self.closed()? {
|
||||
Some(marker) => marker,
|
||||
None => {
|
||||
let mut marker = File::from(openat(
|
||||
&self.key,
|
||||
"closed",
|
||||
OFlags::CREATE | OFlags::EXCL | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)?);
|
||||
regular(&marker, self.root.metadata()?.dev())?;
|
||||
marker.write_all(self.domain.closed())?;
|
||||
marker
|
||||
}
|
||||
};
|
||||
// No admission is released until the permanent denial and every
|
||||
// ancestor entry are durable. A failed/cancelled close never removes it.
|
||||
marker.sync_all()?;
|
||||
self.admission.sync_all()?;
|
||||
self.active.sync_all()?;
|
||||
self.key.sync_all()?;
|
||||
self.seals.sync_all()?;
|
||||
self.root.sync_all()?;
|
||||
flock(&self.admission, FlockOperation::Unlock)?;
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
async fn lock(file: &File, operation: FlockOperation) -> io::Result<()> {
|
||||
tokio::time::timeout(LOCK_TIMEOUT, async {
|
||||
loop {
|
||||
match flock(file, operation) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(Errno::WOULDBLOCK | Errno::INTR) => tokio::time::sleep(POLL_INTERVAL).await,
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "artifact disk lease lock deadline"))?
|
||||
}
|
||||
|
||||
async fn open(root: File, key: [u8; 32], domain: Domain) -> Result<Opened> {
|
||||
tokio::task::spawn_blocking(move || Opened::open(root, key, domain))
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
fn read_error(error: io::Error) -> DiskError {
|
||||
match error.kind() {
|
||||
io::ErrorKind::NotFound => DiskError::other("artifact inventory requires closed PUT and repair domains"),
|
||||
io::ErrorKind::InvalidData | io::ErrorKind::NotADirectory => DiskError::FileCorrupt,
|
||||
io::ErrorKind::TimedOut => DiskError::Timeout,
|
||||
_ if error.raw_os_error() == Some(Errno::LOOP.raw_os_error()) => DiskError::FileCorrupt,
|
||||
_ => DiskError::from(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn existing_directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if metadata.dev() != device || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn existing(root: File, key: [u8; 32], domain: Domain) -> io::Result<Opened> {
|
||||
let device = root.metadata()?.dev();
|
||||
let seals = existing_directory(&root, domain.directory(), device)?;
|
||||
let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
|
||||
let key = existing_directory(&seals, &name, device)?;
|
||||
let lock = |name| -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
&key,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
if regular(&file, device)?.len() != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
};
|
||||
let admission = lock("admission.lock")?;
|
||||
let active = lock("active.lock")?;
|
||||
let opened = Opened {
|
||||
root,
|
||||
seals,
|
||||
key,
|
||||
admission,
|
||||
active,
|
||||
domain,
|
||||
};
|
||||
if opened.closed_with_flags(OFlags::RDONLY)?.is_none() {
|
||||
return Err(io::Error::from(io::ErrorKind::NotFound));
|
||||
}
|
||||
Ok(opened)
|
||||
}
|
||||
|
||||
fn retained(opened: Opened) -> Arc<ArtifactLease> {
|
||||
Arc::new(ArtifactLease {
|
||||
_active: opened.active,
|
||||
_key_directory: opened.key,
|
||||
_seal_directory: opened.seals,
|
||||
_root: opened.root,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn drain_closed(
|
||||
root: File,
|
||||
key: [u8; 32],
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<ArtifactInventoryLease> {
|
||||
let (put, repair) = tokio::task::spawn_blocking(move || {
|
||||
let put = existing(root.try_clone()?, key, Domain::Put)?;
|
||||
let repair = existing(root, key, Domain::Repair)?;
|
||||
Ok::<_, io::Error>((put, repair))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(read_error)?;
|
||||
// Lock order: closed PUT active exclusive -> closed repair active
|
||||
// exclusive. No admission lock is acquired and no seal is created.
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
lock(&put.active, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
lock(&repair.active, FlockOperation::NonBlockingLockExclusive).await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
.map_err(read_error)?;
|
||||
Ok(ArtifactInventoryLease {
|
||||
_put: retained(put),
|
||||
_repair: retained(repair),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn acquire(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
|
||||
let opened = open(root, key, domain).await?;
|
||||
// Lock order: admission exclusive -> active shared. Contention sleeps
|
||||
// asynchronously; no blocking executor thread waits for another owner.
|
||||
lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
tokio::task::spawn_blocking(move || opened.admit())
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
pub(super) async fn seal(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
|
||||
let opened = open(root, key, domain).await?;
|
||||
lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
let opened = tokio::task::spawn_blocking(move || opened.close())
|
||||
.await
|
||||
.map_err(DiskError::from)??;
|
||||
// Closing is already durable. Cancellation or timeout here only loses
|
||||
// the confirmation; admission stays permanently closed across reopen.
|
||||
lock(&opened.active, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
const KEY: [u8; 32] = [7; 32];
|
||||
|
||||
fn root(path: &Path) -> File {
|
||||
use rustix::fs::{Mode, OFlags, open};
|
||||
File::from(
|
||||
open(
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)
|
||||
.expect("open pinned test root"),
|
||||
)
|
||||
}
|
||||
|
||||
fn directory(path: &Path) -> std::path::PathBuf {
|
||||
path.join(Domain::Put.directory())
|
||||
.join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
|
||||
}
|
||||
|
||||
async fn closed(path: &Path) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !directory(path).join("closed").exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("sealer reaches its monotonic close point");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_closing_domain_is_independent_and_survives_reopen() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
seal(root(temp.path()), KEY).await.expect("PUT closes first");
|
||||
let repair = acquire_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("PUT closure permits existing-version repair");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err());
|
||||
drop(repair);
|
||||
seal_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("repair closes independently");
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
seal_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("reopened repair close is durable");
|
||||
|
||||
let other = [8; 32];
|
||||
seal_in(root(temp.path()), other, Domain::Repair)
|
||||
.await
|
||||
.expect("close only repair admission");
|
||||
drop(
|
||||
acquire(root(temp.path()), other)
|
||||
.await
|
||||
.expect("repair closure permits PUT domain"),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_cancelled_waiter_keeps_physical_owner_and_durable_close() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("repair admitted");
|
||||
let (entered, started) = tokio::sync::oneshot::channel();
|
||||
let (release, waiting) = std::sync::mpsc::channel();
|
||||
let owner = tokio::spawn(async move {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = writer;
|
||||
let _ = entered.send(());
|
||||
let _ = waiting.recv();
|
||||
})
|
||||
.await
|
||||
.expect("repair worker");
|
||||
});
|
||||
started.await.expect("physical repair entered");
|
||||
owner.abort();
|
||||
assert!(owner.await.expect_err("cancel repair waiter").is_cancelled());
|
||||
let sealer = tokio::spawn(seal_in(root(temp.path()), KEY, Domain::Repair));
|
||||
let marker = temp
|
||||
.path()
|
||||
.join(Domain::Repair.directory())
|
||||
.join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
|
||||
.join("closed");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !marker.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("repair admission durably closes");
|
||||
assert!(!sealer.is_finished(), "ARTIFACT_REPAIR_CANCELLED_PHYSICAL_TAIL");
|
||||
sealer.abort();
|
||||
assert!(sealer.await.expect_err("cancel repair close waiter").is_cancelled());
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
let pending = seal_in(root(temp.path()), KEY, Domain::Repair);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_REPAIR_REOPEN_MUST_DRAIN_PHYSICAL_TAIL"
|
||||
);
|
||||
release.send(()).expect("release physical repair");
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical drain bounded")
|
||||
.expect("reopened close confirms repair drain");
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_seal_waits_for_independent_openers_and_last_clone() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let first = acquire(root(temp.path()), KEY).await.expect("first admitted writer");
|
||||
let tail = first.clone();
|
||||
let second = acquire(root(temp.path()), KEY).await.expect("independent shared writer");
|
||||
let pending = seal(root(temp.path()), KEY);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_ACTIVE_WRITERS_DRAIN"
|
||||
);
|
||||
drop(first);
|
||||
drop(second);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_PHYSICAL_CLONE_RETAINED"
|
||||
);
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSE_REJECTS_ADMISSION");
|
||||
drop(tail);
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical tail drain bounded")
|
||||
.expect("seal complete");
|
||||
seal(root(temp.path()), KEY)
|
||||
.await
|
||||
.expect("independent reopen preserves closed state");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSED_AFTER_REOPEN");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_close_cancellation_leaves_durable_denial() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("admitted writer");
|
||||
let sealer = tokio::spawn(seal(root(temp.path()), KEY));
|
||||
closed(temp.path()).await;
|
||||
sealer.abort();
|
||||
assert!(sealer.await.expect_err("cancel close waiter").is_cancelled());
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_CANCELLED_CLOSE_STAYS_CLOSED"
|
||||
);
|
||||
drop(writer);
|
||||
seal(root(temp.path()), KEY).await.expect("reopen finishes physical drain");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CANCELLED_CLOSE_REOPEN");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_physical_worker_retains_lease_after_waiter_cancellation() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("physical writer admitted");
|
||||
let (entered, started) = tokio::sync::oneshot::channel();
|
||||
let (release, waiting) = std::sync::mpsc::channel();
|
||||
let owner = tokio::spawn(async move {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _physical_owner = writer;
|
||||
let _ = entered.send(());
|
||||
let _ = waiting.recv();
|
||||
})
|
||||
.await
|
||||
.expect("physical worker");
|
||||
});
|
||||
started.await.expect("physical syscall entered");
|
||||
owner.abort();
|
||||
assert!(owner.await.expect_err("cancel async writer").is_cancelled());
|
||||
let pending = seal(root(temp.path()), KEY);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_CANCELLED_PHYSICAL_TAIL"
|
||||
);
|
||||
release.send(()).expect("release physical worker");
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical worker drains")
|
||||
.expect("seal confirms real completion");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_seal_waits_for_another_process() {
|
||||
const CHILD_ROOT: &str = "RUSTFS_TEST_ARTIFACT_PUT_SEAL_ROOT";
|
||||
if let Some(path) = std::env::var_os(CHILD_ROOT) {
|
||||
seal(root(Path::new(&path)), KEY)
|
||||
.await
|
||||
.expect("child closes and drains parent writer");
|
||||
assert!(acquire(root(Path::new(&path)), KEY).await.is_err(), "ARTIFACT_PUT_CHILD_CLOSED");
|
||||
return;
|
||||
}
|
||||
struct Child(std::process::Child);
|
||||
impl Drop for Child {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("parent owns physical write");
|
||||
let mut child = Child(
|
||||
std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"disk::local::artifact_lease::tests::artifact_put_seal_waits_for_another_process",
|
||||
])
|
||||
.env(CHILD_ROOT, temp.path())
|
||||
.spawn()
|
||||
.expect("start independent sealer"),
|
||||
);
|
||||
closed(temp.path()).await;
|
||||
assert!(child.0.try_wait().expect("probe child").is_none(), "ARTIFACT_PUT_CROSS_PROCESS_DRAIN");
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_CROSS_PROCESS_ADMISSION_CLOSED"
|
||||
);
|
||||
drop(writer);
|
||||
let status = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let Some(status) = child.0.try_wait().expect("child wait") {
|
||||
break status;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("child physically drains");
|
||||
assert!(status.success(), "ARTIFACT_PUT_CROSS_PROCESS_SEAL");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_PROCESS_REOPEN_CLOSED");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_rejects_symlinks_at_every_seal_component() {
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
for component in ["directory", "key", "admission.lock", "active.lock", "closed"] {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let outside = tempfile::tempdir().expect("symlink destination");
|
||||
let sentinel = outside.path().join("untouched");
|
||||
std::fs::write(&sentinel, b"unchanged").expect("sentinel");
|
||||
let target = match component {
|
||||
"directory" => temp.path().join(Domain::Put.directory()),
|
||||
"key" => {
|
||||
std::fs::create_dir(temp.path().join(Domain::Put.directory())).expect("seal directory");
|
||||
std::fs::set_permissions(temp.path().join(Domain::Put.directory()), std::fs::Permissions::from_mode(0o700))
|
||||
.expect("private seal directory");
|
||||
directory(temp.path())
|
||||
}
|
||||
file => {
|
||||
drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
|
||||
let target = directory(temp.path()).join(file);
|
||||
if file != "closed" {
|
||||
std::fs::remove_file(&target).expect("replace fixture lock");
|
||||
}
|
||||
target
|
||||
}
|
||||
};
|
||||
symlink(
|
||||
if matches!(component, "directory" | "key") {
|
||||
outside.path()
|
||||
} else {
|
||||
&sentinel
|
||||
},
|
||||
target,
|
||||
)
|
||||
.expect("symlink fixture");
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_SYMLINK_REJECTED: {component}"
|
||||
);
|
||||
assert!(
|
||||
seal(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_SYMLINK_SEAL_REJECTED: {component}"
|
||||
);
|
||||
assert_eq!(std::fs::read(&sentinel).expect("sentinel intact"), b"unchanged");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_rejects_corrupt_closed_marker() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
|
||||
let marker = directory(temp.path()).join("closed");
|
||||
std::fs::write(&marker, b"partial").expect("corrupt marker fixture");
|
||||
std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o600)).expect("marker permissions");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_FAILS_CLOSED");
|
||||
assert!(seal(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_NOT_CONFIRMED");
|
||||
assert_eq!(std::fs::read(marker).expect("corruption retained"), b"partial");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_stays_on_pinned_root_after_path_replacement() {
|
||||
let temp = tempfile::tempdir().expect("disk root parent");
|
||||
let current = temp.path().join("current");
|
||||
let original = temp.path().join("original");
|
||||
std::fs::create_dir(¤t).expect("initial disk root");
|
||||
let pin = root(¤t);
|
||||
drop(
|
||||
acquire(pin.try_clone().expect("root pin clone"), KEY)
|
||||
.await
|
||||
.expect("initialize pinned seal"),
|
||||
);
|
||||
std::fs::rename(¤t, &original).expect("replace configured pathname");
|
||||
std::fs::create_dir(¤t).expect("replacement disk root");
|
||||
seal(pin, KEY).await.expect("close original disk through its descriptor");
|
||||
assert!(directory(&original).join("closed").exists(), "ARTIFACT_PUT_ROOT_PINNED");
|
||||
assert!(!current.join(Domain::Put.directory()).exists(), "ARTIFACT_PUT_REPLACEMENT_UNTOUCHED");
|
||||
assert!(acquire(root(&original), KEY).await.is_err());
|
||||
}
|
||||
}
|
||||
@@ -345,6 +345,8 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
let publication_root = publication_root.clone();
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
if create_source_parent {
|
||||
#[cfg(test)]
|
||||
super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
|
||||
std::fs::create_dir_all(&source_parent)?;
|
||||
}
|
||||
let guard = match os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root) {
|
||||
@@ -365,9 +367,15 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
};
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let result = {
|
||||
let _ = mutation_lease;
|
||||
if create_source_parent {
|
||||
fs::create_dir_all(source_parent).await.map_err(to_file_error)?;
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
|
||||
std::fs::create_dir_all(source_parent)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
let guard =
|
||||
os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root).map_err(|err| {
|
||||
@@ -398,30 +406,31 @@ pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
prepared: Option<Arc<os::NamespaceMutationLease>>,
|
||||
}
|
||||
|
||||
pub(in crate::disk) struct PreparedRenameData {
|
||||
file_info: FileInfo,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
async fn prepare_rename_data_admission(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
source_volume: Option<&str>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<PreparedRenameData> {
|
||||
let (src_volume, src_path) = source;
|
||||
let (dst_volume, dst_path) = destination;
|
||||
let mut fi = fi;
|
||||
// A non-force DeleteBucket must not remove a directory while a local
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
// optimistic; this lease establishes the local commit/delete order and
|
||||
// remains owned by any blocking syscall that outlives async cancellation.
|
||||
let (dst_volume_dir, destination_object_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
|
||||
let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
let quota_fence_token =
|
||||
match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) {
|
||||
Some(value) => {
|
||||
@@ -455,16 +464,27 @@ impl LocalDisk {
|
||||
.strip_prefix(self.io_root())
|
||||
.map_err(|_| DiskError::InvalidPath)?,
|
||||
);
|
||||
let metadata_lease =
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, state.namespace_owner.take()).await;
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await;
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, namespace_owner).await;
|
||||
let mutation_lease = if let Some(source_volume) = source_volume {
|
||||
os::acquire_rename_data_mutation_lease_for_volumes(
|
||||
&self.root,
|
||||
source_volume,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await
|
||||
};
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -472,6 +492,99 @@ impl LocalDisk {
|
||||
fi.erasure.index = 0;
|
||||
}
|
||||
fi.validate_for_metadata_read()?;
|
||||
Ok(PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn begin_legacy_rename_data(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
permit: crate::disk::io_admission::DiskIoPermit,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<PreparedRenameData> {
|
||||
// Capture the same permit, mount FD and listener-instance owner before
|
||||
// quota -> metadata -> namespace -> sorted volume READ admission.
|
||||
#[cfg(target_os = "linux")]
|
||||
let owner: Arc<dyn Send + Sync> = Arc::new((permit, self.mount_lease.clone(), namespace_owner));
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let owner: Arc<dyn Send + Sync> = Arc::new((permit, namespace_owner));
|
||||
let prepared = self
|
||||
.prepare_rename_data_admission(fi.clone(), source, destination, Some(source.0), Some(owner))
|
||||
.await?;
|
||||
for volume in [Some(source.0), (source.0 != destination.0).then_some(destination.0)]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if !skip_access_checks(volume) {
|
||||
let path = self.io_get_bucket_path(volume)?;
|
||||
let metadata = fs::symlink_metadata(path)
|
||||
.await
|
||||
.map_err(crate::disk::error_conv::to_volume_error)?;
|
||||
if !metadata.is_dir() {
|
||||
return Err(DiskError::VolumeNotFound);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_prepared(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
prepared: PreparedRenameData,
|
||||
) -> Result<RenameDataResp> {
|
||||
let PreparedRenameData {
|
||||
file_info,
|
||||
mutation_lease,
|
||||
} = prepared;
|
||||
let mut state = RenameDataState {
|
||||
prepared: Some(mutation_lease),
|
||||
..Default::default()
|
||||
};
|
||||
self.rename_data_inner(source.0, source.1, file_info, destination.0, destination.1, &mut state)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
} = if let Some(mutation_lease) = state.prepared.take() {
|
||||
PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
}
|
||||
} else {
|
||||
self.prepare_rename_data_admission(
|
||||
fi,
|
||||
(src_volume, src_path),
|
||||
(dst_volume, dst_path),
|
||||
None,
|
||||
state.namespace_owner.take(),
|
||||
)
|
||||
.await?
|
||||
};
|
||||
let no_inline = fi.data.is_none() && fi.size > 0;
|
||||
let fi_healing = fi.is_healing();
|
||||
// Snapshot the destination part paths before `fi` is consumed below. These
|
||||
// are the descriptors a reader may hold for the version this call is about
|
||||
// to replace (backlog#1145); readers build the identical string in
|
||||
@@ -507,6 +620,7 @@ impl LocalDisk {
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(dst_volume)
|
||||
&& let Err(e) = super::super::fs::access_std(&dst_volume_dir)
|
||||
{
|
||||
@@ -599,10 +713,10 @@ impl LocalDisk {
|
||||
|
||||
if preserve_upload_metadata {
|
||||
let source_object_path = self.get_object_path(src_volume, src_path)?;
|
||||
let authority_owner: Arc<dyn Send + Sync> = if source_object_path == root_destination_object_path {
|
||||
let authority_owner: Arc<dyn Send + Sync> = if source_object_path == self.get_object_path(dst_volume, dst_path)? {
|
||||
commit.mutation_lease.clone()
|
||||
} else {
|
||||
os::acquire_metadata_mutation_lease(&source_object_path, Some(commit.mutation_lease.clone())).await
|
||||
os::try_acquire_metadata_mutation_lease_with_owner(&source_object_path, commit.mutation_lease.clone())?
|
||||
};
|
||||
let authority = self.io_get_object_path(src_volume, &format!("{src_path}/{STORAGE_FORMAT_FILE}"))?;
|
||||
let recovery = self.io_get_object_path(
|
||||
@@ -610,14 +724,10 @@ impl LocalDisk {
|
||||
&format!("{src_path}/{}/{STORAGE_FORMAT_FILE}", super::super::MULTIPART_RECOVERY_DIR),
|
||||
)?;
|
||||
check_path_length(recovery.to_string_lossy().as_ref())?;
|
||||
os::rename_all_with_owner(
|
||||
&authority,
|
||||
&recovery,
|
||||
&src_volume_dir,
|
||||
commit.publication_root,
|
||||
Some(authority_owner.clone()),
|
||||
)
|
||||
.await?;
|
||||
// Both volume READs are already held. Neither source key may wait
|
||||
// behind a competing owner that is queued behind a volume WRITE.
|
||||
let recovery_lease = os::try_acquire_namespace_mutation_lease_with_owner(&recovery, authority_owner.clone())?;
|
||||
os::rename_all_with_lease(&authority, &recovery, &src_volume_dir, commit.publication_root, recovery_lease).await?;
|
||||
// This is system metadata authority, so use its existing strict
|
||||
// durability policy even when the object payload is relaxed.
|
||||
if effective_durability(src_volume).syncs_commit_metadata() {
|
||||
@@ -838,7 +948,7 @@ impl LocalDisk {
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
|
||||
let backup_parent = dst_file_parent.join(old_data_dir.to_string());
|
||||
#[cfg(not(windows))]
|
||||
if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await {
|
||||
if let Err(err) = os::make_dir_all_with_owner(&backup_parent, &skip_parent, commit.mutation_lease.clone()).await {
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1305,18 +1415,25 @@ impl LocalDisk {
|
||||
let backup_path = dst_parent
|
||||
.join(rollback_data_dir.to_string())
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
// rename_all acquires the backup path's namespace lease. Do not
|
||||
// hold a disk admission while acquiring another namespace lock.
|
||||
// Keep backup mutations serialized, but never wait for a second
|
||||
// namespace while holding the volume READ: its owner can be
|
||||
// queued behind a volume DELETE that is waiting for this commit.
|
||||
drop(file_sync_admission.take());
|
||||
if let Err(err) = os::rename_all_with_owner(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
commit.publication_root,
|
||||
Some(commit.mutation_lease.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let backup_result =
|
||||
match os::try_acquire_namespace_mutation_lease_with_owner(&backup_path, commit.mutation_lease.clone()) {
|
||||
Ok(lease) => {
|
||||
os::rename_all_with_lease(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
commit.publication_root,
|
||||
lease,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if let Err(err) = backup_result {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
@@ -188,6 +188,7 @@ pub(super) struct DeletePreparationContext {
|
||||
pub transaction: Uuid,
|
||||
pub bucket_incarnation: Option<Uuid>,
|
||||
pub namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub prepared_publication: Option<Arc<os::NamespaceMutationLease>>,
|
||||
}
|
||||
|
||||
pub(super) fn digest(bytes: &[u8]) -> DeleteRecoveryDigest {
|
||||
@@ -1272,6 +1273,7 @@ impl LocalDisk {
|
||||
original: Option<&[u8]>,
|
||||
target: Option<Vec<u8>>,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
prepared_publication: Option<Arc<os::NamespaceMutationLease>>,
|
||||
) -> Result<Option<Vec<u8>>> {
|
||||
let volume = journal.volume.as_str();
|
||||
let object = journal.object.as_str();
|
||||
@@ -1286,9 +1288,12 @@ impl LocalDisk {
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
#[cfg(test)]
|
||||
let record_destination = destination.clone();
|
||||
let staging_root = temporary;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner.clone()).await;
|
||||
let (staged, target) = os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
let staging_root = temporary.clone();
|
||||
let lease = match prepared_publication {
|
||||
Some(publication) => publication,
|
||||
None => os::acquire_namespace_mutation_lease_with_owner(&destination, owner.clone()).await,
|
||||
};
|
||||
let (staged, target) = os::run_blocking_namespace_operation_with_sync(lease.clone(), sync, move || {
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let staged = Arc::new(
|
||||
tempfile::Builder::new()
|
||||
@@ -1327,7 +1332,7 @@ impl LocalDisk {
|
||||
// Proof GC may persist before its final rmdir. Only an empty
|
||||
// leaf can be retired; nested prefixes and new evidence fail
|
||||
// closed under the same namespace ownership as publication.
|
||||
match os::remove_dir_with_owner(&destination, owner.clone()).await {
|
||||
match os::remove_dir_with_lease(&destination, lease.clone()).await {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) if is_dir_not_empty_error(&err) => return Err(DiskError::OutdatedXLMeta),
|
||||
@@ -1340,8 +1345,12 @@ impl LocalDisk {
|
||||
}
|
||||
// The directory owner accompanies the blocking rename, so cancelling
|
||||
// its waiter cannot delete staged files while activation is running.
|
||||
let publication_owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((owner.clone(), staged.clone())));
|
||||
os::rename_all_with_owner(staged.path(), &destination, &bucket, &self.publication_root, publication_owner).await?;
|
||||
// The fresh staging key cannot alias a forward-plan key. Its real
|
||||
// lease retains the destination owner and TempDir through the syscall,
|
||||
// without replacing the prepared publication's sibling guards.
|
||||
let publication_owner: Arc<dyn Send + Sync> = Arc::new((lease, staged.clone()));
|
||||
let activation = os::try_acquire_namespace_mutation_lease_with_owner(staged.path(), publication_owner)?;
|
||||
os::rename_all_with_lease(staged.path(), &destination, &bucket, &self.publication_root, activation).await?;
|
||||
if sync {
|
||||
let mut dirs_to_sync = Vec::new();
|
||||
let mut path = root.as_path();
|
||||
@@ -1381,6 +1390,7 @@ impl LocalDisk {
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
namespace_owner: owner,
|
||||
prepared_publication,
|
||||
} = context;
|
||||
let mut plan = original.map(FileMeta::load).transpose()?.unwrap_or_else(FileMeta::new);
|
||||
let mut deleted_headers: Vec<_> = plan
|
||||
@@ -1456,7 +1466,8 @@ impl LocalDisk {
|
||||
journal.deleted_versions.clone_from(&authority.deleted_versions);
|
||||
}
|
||||
drop(plan);
|
||||
self.activate_delete_preparation(&journal, original, target, owner).await
|
||||
self.activate_delete_preparation(&journal, original, target, owner, prepared_publication)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_recovery_settle(
|
||||
@@ -1475,7 +1486,24 @@ impl LocalDisk {
|
||||
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
|
||||
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
|
||||
self.delete_recovery_settle_owned(volume, object, transaction, expected_fingerprint, action, owner)
|
||||
self.delete_recovery_settle_owned((volume, object), transaction, expected_fingerprint, action, owner, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn delete_recovery_settle_prepared(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
expected_fingerprint: DeleteRecoveryDigest,
|
||||
action: DeleteRecoveryAction,
|
||||
publication: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
if !safe_path(object, false) || transaction.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(publication.clone());
|
||||
self.delete_recovery_settle_owned((volume, object), transaction, expected_fingerprint, action, owner, Some(publication))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1485,6 +1513,7 @@ impl LocalDisk {
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
prepared_publication: Option<Arc<os::NamespaceMutationLease>>,
|
||||
) -> Result<bool> {
|
||||
let prepared = self.io_get_object_path(volume, &format!("{object}/{transaction}/{PREPARED}"))?;
|
||||
if regular_bytes(&prepared)?.is_none() {
|
||||
@@ -1493,20 +1522,28 @@ impl LocalDisk {
|
||||
let state = self
|
||||
.delete_recovery_inspect_owned(volume, object, transaction, owner.clone())
|
||||
.await?;
|
||||
self.delete_recovery_settle_owned(volume, object, transaction, state.fingerprint, DeleteRecoveryAction::Rollback, owner)
|
||||
.await?;
|
||||
self.delete_recovery_settle_owned(
|
||||
(volume, object),
|
||||
transaction,
|
||||
state.fingerprint,
|
||||
DeleteRecoveryAction::Rollback,
|
||||
owner,
|
||||
prepared_publication,
|
||||
)
|
||||
.await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn delete_recovery_settle_owned(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
target: (&str, &str),
|
||||
transaction: Uuid,
|
||||
expected_fingerprint: DeleteRecoveryDigest,
|
||||
action: DeleteRecoveryAction,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
prepared_publication: Option<Arc<os::NamespaceMutationLease>>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
let (volume, object) = target;
|
||||
let (mut observed, mut has_data_dir_work) = self
|
||||
.delete_recovery_inspect_owned_with_markers(volume, object, transaction, owner.clone())
|
||||
.await?;
|
||||
@@ -1534,14 +1571,14 @@ impl LocalDisk {
|
||||
journal: local.clone(),
|
||||
finished: false,
|
||||
};
|
||||
write_delete_rollback_file(
|
||||
write_delete_rollback_file_with_prepared_publication(
|
||||
self.io_root(),
|
||||
volume,
|
||||
&root,
|
||||
(volume, &root),
|
||||
transaction,
|
||||
DELETE_QUORUM_COMMITTED_FILE,
|
||||
&serde_json::to_vec(&receipt)?,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?;
|
||||
@@ -1558,19 +1595,21 @@ impl LocalDisk {
|
||||
witness.target_digest = None;
|
||||
witness.data_dirs.clear();
|
||||
let root = self.io_get_object_path(volume, object)?;
|
||||
let _ = self.activate_delete_preparation(&witness, None, None, owner.clone()).await?;
|
||||
let _ = self
|
||||
.activate_delete_preparation(&witness, None, None, owner.clone(), None)
|
||||
.await?;
|
||||
let receipt = CommitReceipt {
|
||||
journal: witness,
|
||||
finished: false,
|
||||
};
|
||||
write_delete_rollback_file(
|
||||
write_delete_rollback_file_with_prepared_publication(
|
||||
self.io_root(),
|
||||
volume,
|
||||
&root,
|
||||
(volume, &root),
|
||||
transaction,
|
||||
DELETE_QUORUM_COMMITTED_FILE,
|
||||
&serde_json::to_vec(&receipt)?,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?;
|
||||
@@ -1592,6 +1631,7 @@ impl LocalDisk {
|
||||
transaction,
|
||||
bucket_incarnation: authority.bucket_incarnation,
|
||||
namespace_owner: owner.clone(),
|
||||
prepared_publication: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
@@ -1603,14 +1643,14 @@ impl LocalDisk {
|
||||
journal: adopted_journal,
|
||||
finished: false,
|
||||
};
|
||||
write_delete_rollback_file(
|
||||
write_delete_rollback_file_with_prepared_publication(
|
||||
self.io_root(),
|
||||
volume,
|
||||
&root,
|
||||
(volume, &root),
|
||||
transaction,
|
||||
DELETE_QUORUM_COMMITTED_FILE,
|
||||
&serde_json::to_vec(&receipt)?,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?;
|
||||
@@ -1631,6 +1671,7 @@ impl LocalDisk {
|
||||
transaction,
|
||||
bucket_incarnation: None,
|
||||
namespace_owner: owner.clone(),
|
||||
prepared_publication: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
@@ -1672,11 +1713,26 @@ impl LocalDisk {
|
||||
let parent = relative.parent().filter(|parent| !parent.as_os_str().is_empty());
|
||||
let staged_object =
|
||||
parent.map_or_else(|| object.to_owned(), |parent| format!("{object}/{}", parent.to_string_lossy()));
|
||||
self.reserve_version_delete_with_namespace_owner(volume, &staged_object, data_dir, transaction, owner.clone())
|
||||
.await?;
|
||||
self.reserve_version_delete_with_prepared_publication(
|
||||
volume,
|
||||
&staged_object,
|
||||
data_dir,
|
||||
transaction,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
let reserved = dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let private = dir.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
match os::rename_with_namespace_owner(&reserved, &private, owner.clone()).await {
|
||||
match async {
|
||||
if let Some(publication) = &prepared_publication {
|
||||
os::rename_with_lease(reserved.clone(), private.clone(), publication.clone()).await
|
||||
} else {
|
||||
os::rename_with_namespace_owner(&reserved, &private, owner.clone()).await
|
||||
}
|
||||
}
|
||||
.await
|
||||
{
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
@@ -1703,7 +1759,15 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
for file in plan.proof_files {
|
||||
match os::remove_file_with_owner(file, owner.clone()).await {
|
||||
match async {
|
||||
if let Some(publication) = &prepared_publication {
|
||||
os::remove_file_with_lease(file, publication.clone()).await
|
||||
} else {
|
||||
os::remove_file_with_owner(file, owner.clone()).await
|
||||
}
|
||||
}
|
||||
.await
|
||||
{
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
@@ -1743,15 +1807,15 @@ impl LocalDisk {
|
||||
finished: false,
|
||||
};
|
||||
let inspect_after = (!cfg!(feature = "e2e-test-hooks")).then_some(object);
|
||||
let inspected = write_delete_rollback_files(
|
||||
let inspected = write_delete_rollback_files_with_prepared_publication(
|
||||
self.io_root(),
|
||||
volume,
|
||||
&root,
|
||||
(volume, &root),
|
||||
transaction,
|
||||
&[],
|
||||
&[(DELETE_QUORUM_COMMITTED_FILE, &serde_json::to_vec(&receipt)?)],
|
||||
inspect_after,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?;
|
||||
@@ -1769,8 +1833,15 @@ impl LocalDisk {
|
||||
return Ok(observed);
|
||||
}
|
||||
for dir in journal.data_dirs.iter().filter(|_| has_data_dir_work) {
|
||||
self.reserve_version_delete_with_namespace_owner(volume, object, *dir, transaction, owner.clone())
|
||||
.await?;
|
||||
self.reserve_version_delete_with_prepared_publication(
|
||||
volume,
|
||||
object,
|
||||
*dir,
|
||||
transaction,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if observed.current_digest == journal.original_digest {
|
||||
if journal.target_digest.is_some() {
|
||||
@@ -1781,15 +1852,23 @@ impl LocalDisk {
|
||||
&target,
|
||||
true,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
self.delete_file_with_namespace_owner(
|
||||
&self.io_get_bucket_path(volume)?,
|
||||
&root.join(STORAGE_FORMAT_FILE),
|
||||
false,
|
||||
false,
|
||||
self.delete_file_with_prepared_entry(
|
||||
(&self.io_get_bucket_path(volume)?, &root.join(STORAGE_FORMAT_FILE)),
|
||||
(false, false),
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
(
|
||||
None,
|
||||
prepared_publication.as_ref().map(|publication| PreparedPathDelete {
|
||||
lease: publication.clone(),
|
||||
trash: None,
|
||||
encoded_trash: None,
|
||||
}),
|
||||
),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
@@ -1799,19 +1878,33 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
if has_data_dir_work {
|
||||
self.commit_reserved_version_delete_with_namespace_owner(volume, object, transaction, owner.clone())
|
||||
.await?;
|
||||
self.commit_reserved_version_delete_with_publication(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
os::remove_files_and_fsync_with_owner(
|
||||
&rollback,
|
||||
&[STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE],
|
||||
effective_durability(volume).syncs_commit_metadata(),
|
||||
None,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
let (_, complete) = self
|
||||
.finish_version_delete(volume, object, transaction, true, owner.clone())
|
||||
.finish_version_delete_with_prepared_publication(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
true,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
if !complete {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
@@ -1822,17 +1915,26 @@ impl LocalDisk {
|
||||
match fs::symlink_metadata(&staged).await {
|
||||
Ok(metadata) if metadata.is_dir() => {
|
||||
let staged_object = format!("{object}/{transaction}");
|
||||
self.reserve_version_delete_with_namespace_owner(
|
||||
self.reserve_version_delete_with_prepared_publication(
|
||||
volume,
|
||||
&staged_object,
|
||||
*dir,
|
||||
transaction,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
let source = staged.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let destination = staged.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
match os::rename_with_namespace_owner(&source, &destination, owner.clone()).await {
|
||||
match async {
|
||||
if let Some(publication) = &prepared_publication {
|
||||
os::rename_with_lease(source.clone(), destination.clone(), publication.clone()).await
|
||||
} else {
|
||||
os::rename_with_namespace_owner(&source, &destination, owner.clone()).await
|
||||
}
|
||||
}
|
||||
.await
|
||||
{
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
@@ -1874,15 +1976,15 @@ impl LocalDisk {
|
||||
} else {
|
||||
&[STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE]
|
||||
};
|
||||
if let Some(inspected) = write_delete_rollback_files(
|
||||
if let Some(inspected) = write_delete_rollback_files_with_prepared_publication(
|
||||
self.io_root(),
|
||||
volume,
|
||||
&root,
|
||||
(volume, &root),
|
||||
transaction,
|
||||
remove_before_publish,
|
||||
&[(FINISHED, &[]), (DELETE_QUORUM_COMMITTED_FILE, &receipt_bytes)],
|
||||
Some(object),
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -1897,11 +1999,17 @@ impl LocalDisk {
|
||||
let mut file_leases = Vec::with_capacity(3);
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, owner.clone()).await;
|
||||
let lease = match &prepared_publication {
|
||||
Some(publication) => publication.clone(),
|
||||
None => os::acquire_namespace_mutation_lease_with_owner(&path, owner.clone()).await,
|
||||
};
|
||||
file_leases.push((path, lease));
|
||||
}
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let receipt_lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner.clone()).await;
|
||||
let receipt_lease = match &prepared_publication {
|
||||
Some(publication) => publication.clone(),
|
||||
None => os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner.clone()).await,
|
||||
};
|
||||
let prune_paths: Vec<_> = rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
@@ -1909,7 +2017,10 @@ impl LocalDisk {
|
||||
.collect();
|
||||
let mut prune_dirs = Vec::with_capacity(prune_paths.len());
|
||||
for dir in prune_paths {
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&dir, owner.clone()).await;
|
||||
let lease = match &prepared_publication {
|
||||
Some(publication) => os::try_acquire_namespace_mutation_lease_with_owner(&dir, publication.clone())?,
|
||||
None => os::acquire_namespace_mutation_lease_with_owner(&dir, owner.clone()).await,
|
||||
};
|
||||
prune_dirs.push((dir, lease));
|
||||
}
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
@@ -2033,7 +2144,15 @@ impl LocalDisk {
|
||||
}
|
||||
if journal.original_digest.is_none() && journal.target_digest.is_none() {
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
match os::remove_file_with_owner(rollback.join(name), owner.clone()).await {
|
||||
match async {
|
||||
if let Some(publication) = &prepared_publication {
|
||||
os::remove_file_with_lease(rollback.join(name), publication.clone()).await
|
||||
} else {
|
||||
os::remove_file_with_owner(rollback.join(name), owner.clone()).await
|
||||
}
|
||||
}
|
||||
.await
|
||||
{
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
@@ -2051,13 +2170,14 @@ impl LocalDisk {
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
restore_delete_rollback_with_namespace_owner(
|
||||
restore_delete_rollback_with_prepared_publication(
|
||||
volume,
|
||||
&root,
|
||||
&root.join(STORAGE_FORMAT_FILE),
|
||||
transaction,
|
||||
&self.publication_root,
|
||||
owner.clone(),
|
||||
prepared_publication.clone(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -2398,6 +2518,327 @@ mod tests {
|
||||
(temp, disk, info, bytes)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_prepared_batch_rejects_contended_new_namespace_keys() {
|
||||
use crate::disk::io_admission::DiskIoAdmission;
|
||||
|
||||
for blocked in ["backup", "marker", "inspection"] {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
let applied = apply(&disk, transaction).await;
|
||||
let committed = disk
|
||||
.settle_delete_transaction(
|
||||
"recovery",
|
||||
"object",
|
||||
transaction,
|
||||
applied.fingerprint,
|
||||
DeleteRecoveryAction::PublishCommit,
|
||||
)
|
||||
.await
|
||||
.expect("publish the actual delete decision");
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object root");
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let evidence = [
|
||||
root.join(STORAGE_FORMAT_FILE),
|
||||
rollback.join(STORAGE_FORMAT_FILE_BACKUP),
|
||||
rollback.join(DELETE_MARKER_ROLLBACK_FILE),
|
||||
rollback.join(PREPARED),
|
||||
rollback.join(TARGET),
|
||||
rollback.join(FINISHED),
|
||||
rollback.join(DELETE_QUORUM_COMMITTED_FILE),
|
||||
];
|
||||
let before: Vec<_> = evidence
|
||||
.iter()
|
||||
.map(|path| regular_bytes(path).expect("original evidence"))
|
||||
.collect();
|
||||
let admission = DiskIoAdmission::new(disk.io_root().to_string_lossy());
|
||||
let publication = disk
|
||||
.begin_legacy_metadata_write("recovery", "object", admission.foreground(), None, false)
|
||||
.await
|
||||
.expect("real metadata, final xl.meta and volume READ admission");
|
||||
let contended = match blocked {
|
||||
"backup" => rollback.join(STORAGE_FORMAT_FILE_BACKUP),
|
||||
"marker" => rollback.join(DELETE_MARKER_ROLLBACK_FILE),
|
||||
"inspection" => root.clone(),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let contender = os::acquire_namespace_mutation_lease_with_owner(&contended, None).await;
|
||||
let receipt = serde_json::to_vec(&CommitReceipt {
|
||||
journal: committed.journal.expect("committed journal"),
|
||||
finished: true,
|
||||
})
|
||||
.expect("finished receipt bytes");
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
write_delete_rollback_files_with_prepared_publication(
|
||||
disk.io_root(),
|
||||
("recovery", &root),
|
||||
transaction,
|
||||
&[STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE],
|
||||
&[(FINISHED, &[]), (DELETE_QUORUM_COMMITTED_FILE, &receipt)],
|
||||
Some("object"),
|
||||
None,
|
||||
Some(publication),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("a captured volume READ must not wait for a new namespace key");
|
||||
let after: Vec<_> = evidence
|
||||
.iter()
|
||||
.map(|path| regular_bytes(path).expect("preserved evidence"))
|
||||
.collect();
|
||||
drop(contender);
|
||||
assert_eq!(after, before, "{blocked}: contention must reject before any physical write or removal");
|
||||
assert!(
|
||||
matches!(result, Err(DiskError::Io(ref error)) if error.kind() == ErrorKind::WouldBlock),
|
||||
"{blocked}: a different prepared namespace key must return WouldBlock: {result:?}"
|
||||
);
|
||||
|
||||
let publication = disk
|
||||
.begin_legacy_metadata_write("recovery", "object", admission.foreground(), None, false)
|
||||
.await
|
||||
.expect("new prepared admission after contention");
|
||||
let finished = write_delete_rollback_files_with_prepared_publication(
|
||||
disk.io_root(),
|
||||
("recovery", &root),
|
||||
transaction,
|
||||
&[STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE],
|
||||
&[(FINISHED, &[]), (DELETE_QUORUM_COMMITTED_FILE, &receipt)],
|
||||
Some("object"),
|
||||
None,
|
||||
Some(publication),
|
||||
)
|
||||
.await
|
||||
.expect("uncontended physical batch retry")
|
||||
.expect("actual post-publication inspection");
|
||||
assert_eq!(finished.state, DeleteRecoveryState::CommitFinished, "{blocked}");
|
||||
assert_eq!(
|
||||
fs::read(rollback.join(DELETE_QUORUM_COMMITTED_FILE))
|
||||
.await
|
||||
.expect("published receipt"),
|
||||
receipt
|
||||
);
|
||||
assert!(!rollback.join(STORAGE_FORMAT_FILE_BACKUP).exists());
|
||||
assert!(!rollback.join(DELETE_MARKER_ROLLBACK_FILE).exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_prepared_finalize_rejects_contended_prune_then_retries() {
|
||||
use crate::disk::disk_store::LocalDiskWrapper;
|
||||
|
||||
let (_temp, disk, mut remaining, original) = fixture().await;
|
||||
remaining.version_id = Some(Uuid::new_v4());
|
||||
remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1));
|
||||
let mut metadata = FileMeta::load(&original).expect("original metadata");
|
||||
metadata.add_version(remaining).expect("retain an independent inline version");
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object root");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), metadata.marshal_msg().expect("versioned metadata"))
|
||||
.await
|
||||
.expect("original and retained versions");
|
||||
let transaction = Uuid::new_v4();
|
||||
let applied = apply(&disk, transaction).await;
|
||||
let committed = disk
|
||||
.settle_delete_transaction(
|
||||
"recovery",
|
||||
"object",
|
||||
transaction,
|
||||
applied.fingerprint,
|
||||
DeleteRecoveryAction::PublishCommit,
|
||||
)
|
||||
.await
|
||||
.expect("publish decision");
|
||||
let finished = disk
|
||||
.settle_delete_transaction(
|
||||
"recovery",
|
||||
"object",
|
||||
transaction,
|
||||
committed.fingerprint,
|
||||
DeleteRecoveryAction::FinishCommit,
|
||||
)
|
||||
.await
|
||||
.expect("complete physical cleanup before finalization");
|
||||
assert_eq!(finished.state, DeleteRecoveryState::CommitFinished);
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let retained = fs::read(root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("retained version and inline payload");
|
||||
let evidence = [PREPARED, TARGET, FINISHED, DELETE_QUORUM_COMMITTED_FILE];
|
||||
let before: Vec<_> = evidence
|
||||
.iter()
|
||||
.map(|name| regular_bytes(&rollback.join(name)).expect("finished evidence"))
|
||||
.collect();
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(disk.clone(), false);
|
||||
let admitted = wrapper
|
||||
.begin_legacy_metadata_write("recovery", "object")
|
||||
.await
|
||||
.expect("prepared finalization owns metadata, xl.meta and volume READ");
|
||||
let contender = os::acquire_namespace_mutation_lease_with_owner(&rollback, None).await;
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
admitted.settle_delete_transaction(transaction, finished.fingerprint, DeleteRecoveryAction::FinalizeCommit),
|
||||
)
|
||||
.await
|
||||
.expect("prepared finalization must reject a different held prune key without waiting");
|
||||
let after: Vec<_> = evidence
|
||||
.iter()
|
||||
.map(|name| regular_bytes(&rollback.join(name)).expect("preserved evidence"))
|
||||
.collect();
|
||||
drop(contender);
|
||||
assert_eq!(after, before, "prune admission must precede every marker and receipt removal");
|
||||
assert_eq!(
|
||||
fs::read(root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("preserved remaining version"),
|
||||
retained
|
||||
);
|
||||
assert!(
|
||||
matches!(result, Err(DiskError::Io(ref error)) if error.kind() == ErrorKind::WouldBlock),
|
||||
"new prune namespace contention must return WouldBlock: {result:?}"
|
||||
);
|
||||
let admitted = wrapper
|
||||
.begin_legacy_metadata_write("recovery", "object")
|
||||
.await
|
||||
.expect("new prepared admission after prune contention");
|
||||
let finalized = admitted
|
||||
.settle_delete_transaction(transaction, finished.fingerprint, DeleteRecoveryAction::FinalizeCommit)
|
||||
.await
|
||||
.expect("uncontended prepared finalization retries successfully");
|
||||
assert_eq!(finalized.state, DeleteRecoveryState::Absent);
|
||||
assert!(!rollback.exists(), "all transaction evidence is retired");
|
||||
assert_eq!(
|
||||
fs::read(root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("remaining inline payload after proof GC"),
|
||||
retained
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_cancelled_prepared_finalize_retains_actual_guards() {
|
||||
use crate::disk::disk_store::LocalDiskWrapper;
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
use futures::FutureExt;
|
||||
|
||||
let (_temp, disk, mut remaining, original) = fixture().await;
|
||||
remaining.version_id = Some(Uuid::new_v4());
|
||||
remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1));
|
||||
let mut metadata = FileMeta::load(&original).expect("original metadata");
|
||||
metadata.add_version(remaining).expect("retain an independent inline version");
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object root");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), metadata.marshal_msg().expect("versioned metadata"))
|
||||
.await
|
||||
.expect("original and retained versions");
|
||||
let transaction = Uuid::new_v4();
|
||||
let applied = apply(&disk, transaction).await;
|
||||
let committed = disk
|
||||
.settle_delete_transaction(
|
||||
"recovery",
|
||||
"object",
|
||||
transaction,
|
||||
applied.fingerprint,
|
||||
DeleteRecoveryAction::PublishCommit,
|
||||
)
|
||||
.await
|
||||
.expect("publish decision");
|
||||
let finished = disk
|
||||
.settle_delete_transaction(
|
||||
"recovery",
|
||||
"object",
|
||||
transaction,
|
||||
committed.fingerprint,
|
||||
DeleteRecoveryAction::FinishCommit,
|
||||
)
|
||||
.await
|
||||
.expect("finish before proof GC");
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let retained = fs::read(root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("remaining version and inline payload");
|
||||
let receipt = fs::read(rollback.join(DELETE_QUORUM_COMMITTED_FILE))
|
||||
.await
|
||||
.expect("finished receipt");
|
||||
let metadata_key = disk.get_object_path("recovery", "object").expect("actual metadata lock key");
|
||||
let volume_lock = os::disk_volume_mutation_lock(&disk.root, "recovery");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let hook = hooks::install_at(hooks::Stage::Remove, &rollback.join(TARGET), move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(disk.clone(), false);
|
||||
let admitted = wrapper
|
||||
.begin_legacy_metadata_write("recovery", "object")
|
||||
.await
|
||||
.expect("real prepared finalization admission");
|
||||
let mut finalizing = tokio::spawn(async move {
|
||||
admitted
|
||||
.settle_delete_transaction(transaction, finished.fingerprint, DeleteRecoveryAction::FinalizeCommit)
|
||||
.await
|
||||
});
|
||||
let observations = std::panic::AssertUnwindSafe(async {
|
||||
tokio::time::timeout(Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("the actual physical finalization closure reaches TARGET removal")
|
||||
.expect("physical worker entered");
|
||||
finalizing.abort();
|
||||
assert!(
|
||||
(&mut finalizing)
|
||||
.await
|
||||
.expect_err("cancel async finalization waiter")
|
||||
.is_cancelled()
|
||||
);
|
||||
assert!(
|
||||
matches!(os::try_acquire_metadata_mutation_lease_with_owner(&metadata_key, Arc::new(())),
|
||||
Err(DiskError::Io(ref error)) if error.kind() == ErrorKind::WouldBlock),
|
||||
"the physical closure must retain the original metadata transaction"
|
||||
);
|
||||
assert!(
|
||||
volume_lock.clone().try_write_owned().is_err(),
|
||||
"the physical closure must retain its volume READ"
|
||||
);
|
||||
assert!(
|
||||
matches!(os::try_acquire_namespace_mutation_lease_with_owner(&rollback, Arc::new(())),
|
||||
Err(DiskError::Io(ref error)) if error.kind() == ErrorKind::WouldBlock),
|
||||
"the physical closure must retain the newly acquired prune namespace"
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(rollback.join(DELETE_QUORUM_COMMITTED_FILE))
|
||||
.await
|
||||
.expect("receipt while paused"),
|
||||
receipt
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("inline payload while paused"),
|
||||
retained
|
||||
);
|
||||
})
|
||||
.catch_unwind()
|
||||
.await;
|
||||
let _ = release_tx.send(());
|
||||
drop(hook);
|
||||
finalizing.abort();
|
||||
let drained = tokio::time::timeout(Duration::from_secs(5), volume_lock.write_owned())
|
||||
.await
|
||||
.expect("physical finalization must finish before its volume WRITE can acquire");
|
||||
drop(drained);
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
assert!(!rollback.exists(), "physical finalization completes after its waiter was cancelled");
|
||||
assert_eq!(
|
||||
fs::read(root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("remaining payload after cancelled proof GC"),
|
||||
retained
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_recovery_batch_parallelism_preserves_overlapping_key_order() {
|
||||
let opts = DeleteOptions {
|
||||
@@ -2523,6 +2964,112 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_prepared_settle_reuses_admission_with_queued_volume_write() {
|
||||
use crate::disk::disk_store::LocalDiskWrapper;
|
||||
let (_temp, disk, _, original) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
let state = apply(&disk, transaction).await;
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(disk.clone(), false);
|
||||
let admitted = wrapper
|
||||
.begin_legacy_metadata_write("recovery", "object")
|
||||
.await
|
||||
.expect("physical admission before classification");
|
||||
let mut deletion = Box::pin(disk.delete_volume("recovery", true));
|
||||
assert!(
|
||||
matches!(futures::poll!(deletion.as_mut()), std::task::Poll::Pending),
|
||||
"volume WRITE queues behind the actual admitted READ"
|
||||
);
|
||||
let restored = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
admitted.settle_delete_transaction(transaction, state.fingerprint, DeleteRecoveryAction::Rollback),
|
||||
)
|
||||
.await
|
||||
.expect("settlement must reuse its metadata, namespace and volume owner")
|
||||
.expect("prepared rollback succeeds");
|
||||
assert_eq!(restored.state, DeleteRecoveryState::Absent);
|
||||
assert_eq!(
|
||||
disk.read_all("recovery", "object/xl.meta")
|
||||
.await
|
||||
.expect("restored real metadata"),
|
||||
original
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(5), deletion)
|
||||
.await
|
||||
.expect("queued WRITE drains after settlement")
|
||||
.expect("old volume deletion");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_guarded_batch_reuses_admission_with_queued_volume_write() {
|
||||
use crate::disk::disk_store::LocalDiskWrapper;
|
||||
let (_temp, disk, mut info, _) = fixture().await;
|
||||
info.data_dir = Some(Uuid::new_v4());
|
||||
info.metadata.insert(
|
||||
"etag".to_owned(),
|
||||
rustfs_utils::crypto::hex(HashAlgorithm::Md5.hash_encode(info.data.as_ref().expect("inline payload"))),
|
||||
);
|
||||
let metadata_path = disk.io_get_object_path("recovery", "object/xl.meta").expect("metadata path");
|
||||
fs::write(&metadata_path, test_meta(info))
|
||||
.await
|
||||
.expect("canonical inline metadata");
|
||||
let original = fs::read(&metadata_path).await.expect("actual canonical metadata bytes");
|
||||
let current = FileMeta::load(&original)
|
||||
.expect("actual metadata")
|
||||
.into_fileinfo("recovery", "object", "", false, false, false)
|
||||
.expect("actual current generation");
|
||||
let expected_current = CurrentDeleteGeneration::from_file_info(¤t).expect("canonical current generation");
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(disk.clone(), false);
|
||||
let admitted = wrapper
|
||||
.begin_legacy_file_write("recovery")
|
||||
.await
|
||||
.expect("physical volume admission before classification");
|
||||
let mut deletion = Box::pin(disk.delete_volume("recovery", true));
|
||||
assert!(
|
||||
matches!(futures::poll!(deletion.as_mut()), std::task::Poll::Pending),
|
||||
"volume WRITE queues behind admitted batch READ"
|
||||
);
|
||||
let transaction = Uuid::new_v4();
|
||||
let errors = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
admitted.delete_versions_guarded(
|
||||
vec![GuardedFileInfoVersions {
|
||||
group: FileInfoVersions {
|
||||
name: "object".to_owned(),
|
||||
versions: vec![request()],
|
||||
..Default::default()
|
||||
},
|
||||
expected_current,
|
||||
}],
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("guarded executor must not take a second volume READ");
|
||||
assert!(errors.iter().all(Option::is_none), "real guarded publication succeeds: {errors:?}");
|
||||
let state = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("durable guarded journal");
|
||||
assert_eq!(state.state, DeleteRecoveryState::NewPrepared);
|
||||
assert_eq!(state.current_digest, state.journal.as_ref().expect("prepared journal").target_digest);
|
||||
assert_eq!(
|
||||
disk.read_all("recovery", &format!("object/{transaction}/{STORAGE_FORMAT_FILE_BACKUP}"))
|
||||
.await
|
||||
.expect("durable original backup"),
|
||||
original
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(5), deletion)
|
||||
.await
|
||||
.expect("queued WRITE drains after guarded publication")
|
||||
.expect("old volume deletion");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_absent_publish_refreshes_prepared_witness() {
|
||||
let (_temp, disk, _info, _original) = fixture().await;
|
||||
@@ -4338,10 +4885,14 @@ mod tests {
|
||||
} else {
|
||||
fs::write(retained.join("unknown-file"), b"retained").await.unwrap();
|
||||
}
|
||||
assert!(matches!(
|
||||
peer.activate_delete_preparation(&witness, None, None, None).await,
|
||||
Err(DiskError::OutdatedXLMeta)
|
||||
));
|
||||
let owner: Arc<dyn Send + Sync> = os::acquire_metadata_mutation_lease(&root, None).await;
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
peer.activate_delete_preparation(&witness, None, None, Some(owner), None),
|
||||
)
|
||||
.await
|
||||
.expect("activation must reuse its destination lease");
|
||||
assert!(matches!(result, Err(DiskError::OutdatedXLMeta)));
|
||||
assert!(!retained.join(PREPARED).exists());
|
||||
if nested {
|
||||
assert!(retained.join("prefix-child").is_dir());
|
||||
@@ -5274,6 +5825,7 @@ mod tests {
|
||||
transaction,
|
||||
bucket_incarnation: None,
|
||||
namespace_owner: None,
|
||||
prepared_publication: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
+154
-36
@@ -27,6 +27,7 @@ mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub use local::artifact_inventory::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
pub mod os;
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -73,6 +74,10 @@ use crate::cluster::rpc::RemoteDisk;
|
||||
use crate::cluster::rpc::build_internode_data_transport_from_env;
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::disk::disk_store::LocalDiskWrapper;
|
||||
pub use crate::disk::disk_store::{
|
||||
LegacyFileStream, LegacyFileWrite, LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete,
|
||||
LegacyPathDeleteOperation, LegacyRenameData, LegacyRenameFile,
|
||||
};
|
||||
use crate::disk::health_state::RuntimeDriveHealthState;
|
||||
use crate::disk::local::ScanGuard;
|
||||
use bytes::Bytes;
|
||||
@@ -93,6 +98,7 @@ use uuid::Uuid;
|
||||
/// Independent admission and physical ownership for one disk rename.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RenameDataGuards {
|
||||
pub(crate) artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
@@ -234,6 +240,117 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Capture local file-write admission before authoritative classification.
|
||||
pub async fn begin_legacy_file_write(&self, volume: &str) -> Result<LegacyFileWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_file_write(volume).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_file_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyFileWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_file_write_with_namespace_owner(volume, owner).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture metadata publication admission before authoritative classification.
|
||||
pub async fn begin_legacy_metadata_write(&self, volume: &str, path: &str) -> Result<LegacyMetadataWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_metadata_write(volume, path).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_metadata_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyMetadataWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_metadata_write_with_namespace_owner(volume, path, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_legacy_admission_failure(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
error: &DiskError,
|
||||
started: std::time::Instant,
|
||||
) {
|
||||
if let Self::Local(disk) = self {
|
||||
disk.record_legacy_admission_failure(operation, error, started);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_path_delete(
|
||||
&self,
|
||||
volume: &str,
|
||||
operation: LegacyPathDeleteOperation,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<LegacyPathDelete> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_path_delete(volume, operation, owner, deadline).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_part_mutation(
|
||||
&self,
|
||||
source: Option<(&str, &str)>,
|
||||
destination: (&str, &str),
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
operation: LegacyPartOperation,
|
||||
) -> Result<LegacyPartMutation> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_part_mutation(source, destination, namespace_owner, operation)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_rename_file(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
durable: bool,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyRenameFile> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_rename_file(source, destination, durable, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_rename_data(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyRenameData> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_rename_data(source, fi, destination, namespace_owner).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
/// A read-only probe; the physical delete repeats the same leaf checks.
|
||||
pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
|
||||
match self {
|
||||
@@ -1057,13 +1174,27 @@ impl Disk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
|
||||
return self
|
||||
.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await;
|
||||
let bucket_scope = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path);
|
||||
let repair_scope = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path);
|
||||
if let Some(scope) = &bucket_scope {
|
||||
scope.check()?;
|
||||
}
|
||||
if let Some(scope) = &repair_scope {
|
||||
scope.check().map_err(DiskError::other)?;
|
||||
}
|
||||
let repair_lease = if let (Self::Local(disk), Some(scope)) = (self, &repair_scope) {
|
||||
if let Some(identity) = scope.receiver_identity() {
|
||||
scope.check_version(fi.version_id).await.map_err(DiskError::other)?;
|
||||
Some(disk.acquire_artifact_repair_lease(identity.receiver_key).await?)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
scope.check()?;
|
||||
self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, Some(scope))
|
||||
let owner: Option<Arc<dyn Send + Sync>> = (bucket_scope.is_some() || repair_scope.is_some())
|
||||
.then(|| Arc::new((bucket_scope, repair_scope, repair_lease)) as Arc<dyn Send + Sync>);
|
||||
self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, owner)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1074,10 +1205,20 @@ impl Disk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: RenameDataGuards,
|
||||
mut guards: RenameDataGuards,
|
||||
) -> RenameDataObservation {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
if let Some(identity) = guards.artifact_put.take() {
|
||||
let lease = match local_disk.acquire_artifact_put_lease(identity.receiver_key).await {
|
||||
Ok(lease) => lease,
|
||||
Err(error) => return RenameDataObservation::unknown(Err(error)),
|
||||
};
|
||||
// Keep the existing timeout policy; physical workers own the
|
||||
// receiver lease even after the caller's disk deadline.
|
||||
guards.namespace_owner =
|
||||
Some(Arc::new((guards.external_guard.clone(), guards.namespace_owner.take(), lease)));
|
||||
}
|
||||
local_disk
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
@@ -1085,39 +1226,17 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
(src_volume, src_path),
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
(dst_volume, dst_path),
|
||||
guards.scanner_publication_lease_token,
|
||||
guards.artifact_put.map(|identity| identity.incarnation),
|
||||
)
|
||||
.await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence_and_guard(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
scanner_publication_lease_token,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn rename_data_borrowed_with_fence_and_guard(
|
||||
&self,
|
||||
@@ -1138,12 +1257,11 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
(src_volume, src_path),
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
(dst_volume, dst_path),
|
||||
scanner_publication_lease_token,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
+553
-52
@@ -29,7 +29,8 @@ use std::{
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio::sync::{
|
||||
Mutex as AsyncMutex, OwnedMutexGuard, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit, oneshot,
|
||||
Mutex as AsyncMutex, OwnedMutexGuard, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, OwnedSemaphorePermit, RwLock, Semaphore,
|
||||
SemaphorePermit, oneshot,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
@@ -273,10 +274,41 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
RemoveComplete,
|
||||
StreamOpen,
|
||||
StreamOpenComplete,
|
||||
StreamWrite,
|
||||
StreamWriteComplete,
|
||||
StreamFlush,
|
||||
StreamFlushComplete,
|
||||
StreamShutdown,
|
||||
StreamShutdownComplete,
|
||||
#[cfg(test)]
|
||||
StreamDrainWaiting,
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
DirectOpened,
|
||||
#[cfg(test)]
|
||||
Rollback,
|
||||
#[cfg(test)]
|
||||
DirFsync,
|
||||
#[cfg(test)]
|
||||
FileSync,
|
||||
#[cfg(test)]
|
||||
PartPrepare,
|
||||
#[cfg(test)]
|
||||
PartRestore,
|
||||
#[cfg(test)]
|
||||
PartCleanup,
|
||||
#[cfg(test)]
|
||||
PartSettle,
|
||||
#[cfg(test)]
|
||||
PartSettleTimeout,
|
||||
#[cfg(test)]
|
||||
PartSettlePlanned,
|
||||
#[cfg(test)]
|
||||
PartSettleComplete,
|
||||
#[cfg(test)]
|
||||
NamespaceWaiting,
|
||||
}
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
@@ -357,6 +389,8 @@ pub struct LocalPublicationPause {
|
||||
_hook: prepared_publication_test_hooks::Guard,
|
||||
entered: oneshot::Receiver<()>,
|
||||
_release: std::sync::mpsc::Sender<()>,
|
||||
_completion_hook: Option<prepared_publication_test_hooks::Guard>,
|
||||
completed: Option<oneshot::Receiver<()>>,
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
@@ -365,6 +399,8 @@ pub enum LocalPublicationStage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
StreamWrite,
|
||||
StreamShutdown,
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
@@ -377,7 +413,26 @@ impl LocalPublicationPause {
|
||||
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
|
||||
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
|
||||
LocalPublicationStage::StreamWrite => prepared_publication_test_hooks::Stage::StreamWrite,
|
||||
LocalPublicationStage::StreamShutdown => prepared_publication_test_hooks::Stage::StreamShutdown,
|
||||
};
|
||||
let complete_stage = match stage {
|
||||
prepared_publication_test_hooks::Stage::StreamWrite => {
|
||||
Some(prepared_publication_test_hooks::Stage::StreamWriteComplete)
|
||||
}
|
||||
prepared_publication_test_hooks::Stage::StreamShutdown => {
|
||||
Some(prepared_publication_test_hooks::Stage::StreamShutdownComplete)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let completion = complete_stage.map(|stage| {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
|
||||
let _ = tx.send(());
|
||||
});
|
||||
(hook, rx)
|
||||
});
|
||||
let (completion_hook, completed) = completion.map_or((None, None), |(hook, rx)| (Some(hook), Some(rx)));
|
||||
let (entered_tx, entered) = oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
|
||||
@@ -388,9 +443,27 @@ impl LocalPublicationPause {
|
||||
_hook: hook,
|
||||
entered,
|
||||
_release: release,
|
||||
_completion_hook: completion_hook,
|
||||
completed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Release a stream barrier and observe its actual physical operation finish.
|
||||
pub async fn release_and_wait(self) -> std::result::Result<(), oneshot::error::RecvError> {
|
||||
let Self {
|
||||
_hook,
|
||||
entered: _,
|
||||
_release,
|
||||
_completion_hook,
|
||||
completed,
|
||||
} = self;
|
||||
drop(_release);
|
||||
if let Some(completed) = completed {
|
||||
completed.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn entered(&mut self) -> std::result::Result<(), oneshot::error::RecvError> {
|
||||
(&mut self.entered).await
|
||||
}
|
||||
@@ -1614,7 +1687,7 @@ fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'stati
|
||||
}
|
||||
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
type NamespaceMutationLock = AsyncMutex<()>;
|
||||
type NamespaceMutationLock = RwLock<()>;
|
||||
type NamespaceMutationLockRegistry = HashMap<PathBuf, Weak<NamespaceMutationLock>>;
|
||||
static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
@@ -1683,7 +1756,7 @@ impl DiskBlockingLane {
|
||||
struct DiskBlockingQueueState {
|
||||
jobs: VecDeque<Box<dyn FnOnce() + Send>>,
|
||||
active_workers: usize,
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
paused_workers: usize,
|
||||
}
|
||||
|
||||
@@ -1713,7 +1786,7 @@ struct DiskBlockingWorkerGuard {
|
||||
|
||||
impl Drop for DiskBlockingWorkerGuard {
|
||||
fn drop(&mut self) {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
if self.state != DiskBlockingWorkerState::Submitted {
|
||||
CURRENT_DISK_BLOCKING_QUEUE.with(|slot| {
|
||||
let _ = slot.borrow_mut().take();
|
||||
@@ -1772,11 +1845,11 @@ impl DiskBlockingQueue {
|
||||
|
||||
#[inline]
|
||||
fn effective_max_workers(&self, state: &DiskBlockingQueueState) -> usize {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
{
|
||||
MAX_NAMESPACE_WORKERS_PER_DISK.saturating_add(state.paused_workers)
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
#[cfg(not(any(test, feature = "test-util", feature = "e2e-test-hooks")))]
|
||||
{
|
||||
let _ = state;
|
||||
MAX_NAMESPACE_WORKERS_PER_DISK
|
||||
@@ -1828,7 +1901,7 @@ impl DiskBlockingQueue {
|
||||
};
|
||||
let worker = move || {
|
||||
let mut guard = guard;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
CURRENT_DISK_BLOCKING_QUEUE.with(|slot| {
|
||||
*slot.borrow_mut() = Some((Arc::clone(&queue), lane));
|
||||
});
|
||||
@@ -1925,18 +1998,18 @@ impl DiskBlockingQueue {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
thread_local! {
|
||||
static CURRENT_DISK_BLOCKING_QUEUE: std::cell::RefCell<Option<(Arc<DiskBlockingQueue>, DiskBlockingLane)>> =
|
||||
const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
pub(crate) struct BlockingTestHookGuard {
|
||||
queue: Option<(Arc<DiskBlockingQueue>, DiskBlockingLane)>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
impl Drop for BlockingTestHookGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some((queue, lane)) = self.queue.take() {
|
||||
@@ -1946,7 +2019,7 @@ impl Drop for BlockingTestHookGuard {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
pub(crate) fn enter_blocking_test_hook() -> BlockingTestHookGuard {
|
||||
let queue = CURRENT_DISK_BLOCKING_QUEUE.with(|slot| slot.borrow().clone());
|
||||
if let Some((ref queue, lane)) = queue {
|
||||
@@ -2033,12 +2106,12 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
|
||||
return lock;
|
||||
}
|
||||
|
||||
let lock = Arc::new(AsyncMutex::new(()));
|
||||
let lock = Arc::new(RwLock::new(()));
|
||||
locks.insert(path.to_path_buf(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
|
||||
static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
|
||||
static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<AsyncMutex<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
/// Serializes the complete xl.meta read/modify/commit transaction. This domain
|
||||
@@ -2049,21 +2122,36 @@ pub(crate) struct MetadataMutationLease {
|
||||
_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
fn disk_metadata_mutation_lock(object: &Path) -> Arc<AsyncMutex<()>> {
|
||||
let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
|
||||
locks.retain(|_, lock| lock.strong_count() > 0);
|
||||
if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
|
||||
lock
|
||||
} else {
|
||||
let lock = Arc::new(AsyncMutex::new(()));
|
||||
locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::disk) fn try_acquire_metadata_mutation_lease_with_owner(
|
||||
object: &Path,
|
||||
owner: Arc<dyn Send + Sync>,
|
||||
) -> Result<Arc<MetadataMutationLease>> {
|
||||
let guard = disk_metadata_mutation_lock(object)
|
||||
.try_lock_owned()
|
||||
.map_err(|_| DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock)))?;
|
||||
Ok(Arc::new(MetadataMutationLease {
|
||||
_guard: guard,
|
||||
_owner: Some(owner),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_metadata_mutation_lease(
|
||||
object: &Path,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<MetadataMutationLease> {
|
||||
let lock = {
|
||||
let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
|
||||
locks.retain(|_, lock| lock.strong_count() > 0);
|
||||
if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
|
||||
lock
|
||||
} else {
|
||||
let lock = Arc::new(AsyncMutex::new(()));
|
||||
locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
};
|
||||
let lock = disk_metadata_mutation_lock(object);
|
||||
Arc::new(MetadataMutationLease {
|
||||
_guard: lock.lock_owned().await,
|
||||
_owner: owner,
|
||||
@@ -2073,9 +2161,10 @@ pub(crate) async fn acquire_metadata_mutation_lease(
|
||||
/// Keeps a namespace transaction serialized even when its async waiter is
|
||||
/// cancelled while a blocking filesystem call is still running.
|
||||
pub(crate) struct NamespaceMutationLease {
|
||||
_namespace_guard: OwnedMutexGuard<()>,
|
||||
_staging_guard: Option<OwnedMutexGuard<()>>,
|
||||
_namespace_guard: OwnedRwLockWriteGuard<()>,
|
||||
_staging_guard: Option<OwnedRwLockWriteGuard<()>>,
|
||||
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||
_secondary_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
|
||||
// Independent of the quota claim; both survive cancellation of the waiter.
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
@@ -2098,15 +2187,44 @@ pub(in crate::disk) async fn acquire_namespace_mutation_lease_with_owner(
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
let blocking_queue = disk_blocking_queue_for_path(path);
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
|
||||
_namespace_guard: disk_namespace_mutation_lock(path).write_owned().await,
|
||||
_staging_guard: None,
|
||||
_volume_guard: None,
|
||||
_secondary_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
blocking_queue,
|
||||
})
|
||||
}
|
||||
|
||||
/// A transaction holding a volume READ must not queue for another namespace:
|
||||
/// its owner may already be waiting behind a queued volume WRITE.
|
||||
pub(in crate::disk) fn try_acquire_namespace_mutation_lease_with_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Arc<dyn Send + Sync>,
|
||||
) -> Result<Arc<NamespaceMutationLease>> {
|
||||
let guard = disk_namespace_mutation_lock(path)
|
||||
.try_write_owned()
|
||||
.map_err(|_| DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock)))?;
|
||||
Ok(Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: guard,
|
||||
_staging_guard: None,
|
||||
_volume_guard: None,
|
||||
_secondary_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: Some(namespace_owner),
|
||||
blocking_queue: disk_blocking_queue_for_path(path),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Protect a parent directory against replacement while sibling parts publish.
|
||||
/// Like exact-path acquisition, this must not wait while holding a volume READ.
|
||||
pub(in crate::disk) fn try_acquire_namespace_parent_read(path: &Path) -> Result<OwnedRwLockReadGuard<()>> {
|
||||
disk_namespace_mutation_lock(path)
|
||||
.try_read_owned()
|
||||
.map_err(|_| DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock)))
|
||||
}
|
||||
|
||||
/// Acquire object serialization before the volume read lock. Bucket deletion
|
||||
/// only acquires the volume write lock, so this order cannot form a lock cycle.
|
||||
pub(crate) async fn acquire_rename_data_mutation_lease(
|
||||
@@ -2129,14 +2247,14 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
let destination_lock = disk_namespace_mutation_lock(destination_object);
|
||||
let (namespace_guard, staging_guard) = match staging_prefix {
|
||||
Some(source) if source < destination_object => {
|
||||
let staging = disk_namespace_mutation_lock(source).lock_owned().await;
|
||||
(destination_lock.lock_owned().await, Some(staging))
|
||||
let staging = disk_namespace_mutation_lock(source).write_owned().await;
|
||||
(destination_lock.write_owned().await, Some(staging))
|
||||
}
|
||||
Some(source) if source > destination_object => {
|
||||
let destination = destination_lock.lock_owned().await;
|
||||
(destination, Some(disk_namespace_mutation_lock(source).lock_owned().await))
|
||||
let destination = destination_lock.write_owned().await;
|
||||
(destination, Some(disk_namespace_mutation_lock(source).write_owned().await))
|
||||
}
|
||||
_ => (destination_lock.lock_owned().await, None),
|
||||
_ => (destination_lock.write_owned().await, None),
|
||||
};
|
||||
let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
|
||||
let blocking_queue = disk_blocking_queue_for_root(root);
|
||||
@@ -2144,12 +2262,79 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
_namespace_guard: namespace_guard,
|
||||
_staging_guard: staging_guard,
|
||||
_volume_guard: Some(volume_guard),
|
||||
_secondary_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
blocking_queue,
|
||||
})
|
||||
}
|
||||
|
||||
/// Admit both physical volumes after the destination namespace, never taking
|
||||
/// the same READ twice when a bucket deletion is already queued.
|
||||
pub(in crate::disk) async fn acquire_rename_data_mutation_lease_for_volumes(
|
||||
root: &Path,
|
||||
source_volume: &str,
|
||||
destination_volume: &str,
|
||||
destination_object: &Path,
|
||||
staging_prefix: Option<&Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
let staging_guard = match staging_prefix {
|
||||
Some(source) if source < destination_object => Some(disk_namespace_mutation_lock(source).write_owned().await),
|
||||
_ => None,
|
||||
};
|
||||
let namespace = disk_namespace_mutation_lock(destination_object).write_owned();
|
||||
#[cfg(all(test, not(windows)))]
|
||||
let namespace_guard = {
|
||||
// Observe actual namespace contention, not a cooperative-budget yield.
|
||||
let mut namespace = std::pin::pin!(tokio::task::unconstrained(namespace));
|
||||
std::future::poll_fn(|cx| {
|
||||
let poll = std::future::Future::poll(namespace.as_mut(), cx);
|
||||
if poll.is_pending() {
|
||||
prepared_publication_test_hooks::run(
|
||||
prepared_publication_test_hooks::Stage::NamespaceWaiting,
|
||||
destination_object,
|
||||
);
|
||||
}
|
||||
poll
|
||||
})
|
||||
.await
|
||||
};
|
||||
#[cfg(not(all(test, not(windows))))]
|
||||
let namespace_guard = namespace.await;
|
||||
let staging_guard = match staging_prefix {
|
||||
Some(source) if source > destination_object => Some(disk_namespace_mutation_lock(source).write_owned().await),
|
||||
_ => staging_guard,
|
||||
};
|
||||
let source_key = root.join(source_volume);
|
||||
let destination_key = root.join(destination_volume);
|
||||
let (first, second) = if source_key <= destination_key {
|
||||
(source_volume, destination_volume)
|
||||
} else {
|
||||
(destination_volume, source_volume)
|
||||
};
|
||||
let first_guard = disk_volume_mutation_lock(root, first).read_owned().await;
|
||||
let second_guard = if source_key == destination_key {
|
||||
None
|
||||
} else {
|
||||
Some(disk_volume_mutation_lock(root, second).read_owned().await)
|
||||
};
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: namespace_guard,
|
||||
_staging_guard: staging_guard,
|
||||
_volume_guard: Some(first_guard),
|
||||
_secondary_volume_guard: second_guard,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
blocking_queue: disk_blocking_queue_for_root(root),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(windows)))]
|
||||
pub(crate) async fn drain_namespace_for_test(path: &Path) {
|
||||
drop(acquire_namespace_mutation_lease_with_owner(path, None).await);
|
||||
}
|
||||
|
||||
/// Always acquire the per-disk permit before the process-wide permit. Keeping
|
||||
/// this order uniform prevents one slow disk from reserving global capacity
|
||||
/// while it waits for its own concurrency slot.
|
||||
@@ -2382,6 +2567,8 @@ pub(crate) mod file_sync_probe {
|
||||
}
|
||||
|
||||
pub(crate) fn sync_file(path: &Path) -> io::Result<()> {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::FileSync, path);
|
||||
#[cfg(test)]
|
||||
let _probe = file_sync_probe::enter(path);
|
||||
#[cfg(test)]
|
||||
@@ -2536,6 +2723,15 @@ pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Keep the physical directory workers tied to their volume mutation owner.
|
||||
pub(crate) async fn make_dir_all_with_owner(path: &Path, base_dir: &Path, owner: Arc<dyn Send + Sync>) -> Result<()> {
|
||||
check_path_length(path.to_string_lossy().as_ref())?;
|
||||
reliable_mkdir_all_with_owner(path, base_dir, Some(owner))
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if a directory is empty.
|
||||
/// Only reads one entry to determine if the directory is empty.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
@@ -2700,6 +2896,8 @@ pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Ar
|
||||
queue
|
||||
.run_fsync_cancellable(move |_| {
|
||||
let _namespace_owner = namespace_owner;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &path);
|
||||
fsync_dir_std(path)
|
||||
})
|
||||
.await
|
||||
@@ -2731,6 +2929,10 @@ pub(crate) async fn remove_file_with_owner(
|
||||
.await;
|
||||
}
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
remove_file_with_lease(path, lease).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn remove_file_with_lease(path: PathBuf, lease: Arc<NamespaceMutationLease>) -> io::Result<()> {
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
@@ -2745,18 +2947,25 @@ pub(crate) async fn remove_files_and_fsync_with_owner(
|
||||
sync: bool,
|
||||
final_remove: Option<&str>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
prepared: Option<Arc<NamespaceMutationLease>>,
|
||||
) -> io::Result<()> {
|
||||
let mut leased_paths = Vec::with_capacity(names.len());
|
||||
// Follow caller order, keeping FINISHED before the final quorum receipt
|
||||
// to match rollback publication's overlapping namespace leases.
|
||||
for name in names {
|
||||
let path = dir.join(name);
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner.clone()).await;
|
||||
let lease = match &prepared {
|
||||
Some(publication) => publication.clone(),
|
||||
None => acquire_namespace_mutation_lease_with_owner(&path, namespace_owner.clone()).await,
|
||||
};
|
||||
leased_paths.push((path, lease));
|
||||
}
|
||||
let final_leased = if let Some(name) = final_remove {
|
||||
let path = dir.join(name);
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner.clone()).await;
|
||||
let lease = match &prepared {
|
||||
Some(publication) => publication.clone(),
|
||||
None => acquire_namespace_mutation_lease_with_owner(&path, namespace_owner.clone()).await,
|
||||
};
|
||||
Some((path, lease))
|
||||
} else {
|
||||
None
|
||||
@@ -2768,6 +2977,9 @@ pub(crate) async fn remove_files_and_fsync_with_owner(
|
||||
.map_or_else(|| disk_blocking_queue_for_path(dir), |(_, lease)| Arc::clone(&lease.blocking_queue));
|
||||
let operation = move |_: &dyn Fn() -> bool| {
|
||||
let _namespace_owner = namespace_owner;
|
||||
// A prepared transaction already owns these keys. Keep its complete
|
||||
// guard set through cleanup, fsync and final receipt removal.
|
||||
let _prepared = prepared;
|
||||
for (path, lease) in leased_paths {
|
||||
let _lease = lease;
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
@@ -2843,6 +3055,10 @@ pub(in crate::disk) async fn rename_with_namespace_owner(
|
||||
let src = src.to_path_buf();
|
||||
let dst = dst.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&dst, namespace_owner).await;
|
||||
rename_with_lease(src, dst, lease).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_with_lease(src: PathBuf, dst: PathBuf, lease: Arc<NamespaceMutationLease>) -> io::Result<()> {
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
{
|
||||
@@ -2863,6 +3079,10 @@ pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
create_dir_all_with_lease(path, lease).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn create_dir_all_with_lease(path: PathBuf, lease: Arc<NamespaceMutationLease>) -> io::Result<()> {
|
||||
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
|
||||
}
|
||||
|
||||
@@ -3188,9 +3408,13 @@ pub(crate) async fn rename_all_with_commit_guard(
|
||||
{
|
||||
let source = src_file_path.as_ref().to_path_buf();
|
||||
let destination = dst_file_path.as_ref().to_path_buf();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let mut physical_barrier = PhysicalRenameTestBarrier::selected(publication_root.path(), &destination, "waiter-dropped")?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let physical_barrier_path = physical_barrier.as_ref().map(|barrier| barrier.path.clone());
|
||||
let guard = commit_guard.clone();
|
||||
let _ = (base_dir, publication_root);
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let result = run_blocking_namespace_operation(lease, move || {
|
||||
guard.directories.prepare_destination()?;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run_rename_destination(&source, &destination);
|
||||
@@ -3199,11 +3423,20 @@ pub(crate) async fn rename_all_with_commit_guard(
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &source);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination);
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(path) = physical_barrier_path.as_ref() {
|
||||
PhysicalRenameTestBarrier::wait(path)?;
|
||||
}
|
||||
guard.directories.rename(&source, &destination, None)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
.map_err(DiskError::from);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(observer) = physical_barrier.as_mut() {
|
||||
observer.complete();
|
||||
}
|
||||
result
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
@@ -3235,8 +3468,18 @@ pub(crate) async fn rename_all_ignore_missing_source_with_owner(
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
rename_all_ignore_missing_source_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_all_ignore_missing_source_with_lease(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
match reliable_rename_inner_with_lease(
|
||||
src_file_path.to_path_buf(),
|
||||
dst_file_path.as_ref().to_path_buf(),
|
||||
@@ -3304,6 +3547,97 @@ async fn reliable_rename_inner(
|
||||
.await
|
||||
}
|
||||
|
||||
// The barrier runs inside the physical worker. Its async observer owns no lease,
|
||||
// and separately records cancellation so E2E tests cannot mistake an open RPC
|
||||
// waiter for ownership retained by a detached filesystem operation.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(in crate::disk) struct PhysicalRenameTestBarrier {
|
||||
path: PathBuf,
|
||||
completed: bool,
|
||||
cancellation_stage: &'static str,
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
impl PhysicalRenameTestBarrier {
|
||||
pub(in crate::disk) fn selected(
|
||||
root: &Path,
|
||||
destination: &Path,
|
||||
cancellation_stage: &'static str,
|
||||
) -> io::Result<Option<Self>> {
|
||||
let path = root.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
let expected = match std::fs::read_to_string(&path) {
|
||||
Ok(expected) => expected,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let Some(parent) = destination.parent() else { return Ok(None) };
|
||||
let Some(name) = destination.file_name() else { return Ok(None) };
|
||||
let Ok(parent) = parent.canonicalize() else { return Ok(None) };
|
||||
let destination = parent.join(name);
|
||||
// Construct lazily: dropping an unselected observer must never report
|
||||
// cancellation of another operation using this disk's barrier file.
|
||||
Ok((destination == Path::new(&expected)).then(|| Self {
|
||||
path,
|
||||
completed: false,
|
||||
cancellation_stage,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(in crate::disk) fn complete(&mut self) {
|
||||
self.completed = true;
|
||||
}
|
||||
|
||||
fn wait(path: &Path) -> io::Result<()> {
|
||||
// The synthetic pause retains physical ownership while unrelated disk work progresses.
|
||||
let _pause = enter_blocking_test_hook();
|
||||
std::fs::write(path.with_extension("arrived"), b"physical worker entered")?;
|
||||
let deadline = Instant::now() + Duration::from_secs(120);
|
||||
while !path.with_extension("release").try_exists()? {
|
||||
if Instant::now() >= deadline {
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "physical rename test barrier was not released"));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
impl Drop for PhysicalRenameTestBarrier {
|
||||
fn drop(&mut self) {
|
||||
if !self.completed {
|
||||
let _ = std::fs::write(self.path.with_extension(self.cancellation_stage), b"async waiter dropped");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "e2e-test-hooks"))]
|
||||
#[test]
|
||||
fn physical_rename_barrier_reports_only_selected_cancelled_waiters() {
|
||||
let root = tempfile::tempdir().expect("physical barrier fixture");
|
||||
let root_path = root.path().canonicalize().expect("canonical fixture root");
|
||||
let barrier = root_path.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
std::fs::create_dir_all(barrier.parent().expect("barrier parent")).expect("barrier directory");
|
||||
let destination = root_path.join("selected");
|
||||
std::fs::write(&barrier, destination.to_str().expect("fixture path")).expect("select physical destination");
|
||||
let marker = barrier.with_extension("waiter-dropped");
|
||||
assert!(
|
||||
PhysicalRenameTestBarrier::selected(&root_path, &root_path.join("other"), "waiter-dropped")
|
||||
.expect("unselected observer")
|
||||
.is_none()
|
||||
);
|
||||
assert!(!marker.exists(), "unselected rename falsely reported cancellation");
|
||||
drop(PhysicalRenameTestBarrier::selected(&root_path, &destination, "waiter-dropped").expect("cancelled observer"));
|
||||
assert!(marker.exists(), "selected cancelled waiter was not observed");
|
||||
std::fs::remove_file(&marker).expect("reset observation");
|
||||
let mut observer = PhysicalRenameTestBarrier::selected(&root_path, &destination, "waiter-dropped")
|
||||
.expect("completed observer")
|
||||
.expect("selected destination");
|
||||
observer.complete();
|
||||
drop(observer);
|
||||
assert!(!marker.exists(), "completed waiter falsely reported cancellation");
|
||||
}
|
||||
|
||||
async fn reliable_rename_inner_with_lease(
|
||||
src_file_path: PathBuf,
|
||||
dst_file_path: PathBuf,
|
||||
@@ -3312,6 +3646,10 @@ async fn reliable_rename_inner_with_lease(
|
||||
warn_on_missing_source: bool,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> io::Result<()> {
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let mut physical_barrier = PhysicalRenameTestBarrier::selected(publication_root.path(), &dst_file_path, "waiter-dropped")?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let physical_barrier_path = physical_barrier.as_ref().map(|barrier| barrier.path.clone());
|
||||
let operation = {
|
||||
let src_file_path = src_file_path.clone();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
@@ -3325,10 +3663,19 @@ async fn reliable_rename_inner_with_lease(
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(path) = physical_barrier_path.as_ref() {
|
||||
PhysicalRenameTestBarrier::wait(path)?;
|
||||
}
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
};
|
||||
let result = run_blocking_namespace_operation(lease, operation).await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(barrier) = physical_barrier.as_mut() {
|
||||
barrier.complete();
|
||||
}
|
||||
|
||||
if let Err(err) = &result
|
||||
&& (warn_on_missing_source || err.kind() != io::ErrorKind::NotFound)
|
||||
{
|
||||
@@ -5182,7 +5529,9 @@ pub(crate) fn compare_and_update_control_file(
|
||||
validate_windows_owned_file(&lock)?;
|
||||
match lock.as_file().try_lock() {
|
||||
Ok(()) => {}
|
||||
Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
|
||||
Err(std::fs::TryLockError::WouldBlock) => {
|
||||
return Err(io::Error::from(io::ErrorKind::WouldBlock));
|
||||
}
|
||||
Err(std::fs::TryLockError::Error(err)) => return Err(err),
|
||||
}
|
||||
|
||||
@@ -5612,11 +5961,15 @@ fn should_retry_prepare(err: &io::Error, attempt: usize, prune_budget: usize) ->
|
||||
}
|
||||
|
||||
pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
reliable_mkdir_all_with_owner(path.as_ref(), base_dir.as_ref(), None).await
|
||||
}
|
||||
|
||||
async fn reliable_mkdir_all_with_owner(path: &Path, base_dir: &Path, owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
let mut i = 0;
|
||||
|
||||
let mut base_dir = base_dir.as_ref();
|
||||
let mut base_dir = base_dir;
|
||||
loop {
|
||||
if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await {
|
||||
if let Err(e) = os_mkdir_all(path, base_dir, owner.clone()).await {
|
||||
if e.kind() == io::ErrorKind::NotFound && i == 0 {
|
||||
i += 1;
|
||||
|
||||
@@ -5638,16 +5991,34 @@ pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Pat
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mkdir_with_owner(path: &Path, recursive: bool, owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
let path = path.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = owner;
|
||||
#[cfg(test)]
|
||||
super::local::volume_mutation_test_hooks::run(super::local::volume_mutation_test_hooks::Stage::Create, &path);
|
||||
let result = if recursive {
|
||||
std::fs::create_dir_all(&path)
|
||||
} else {
|
||||
std::fs::create_dir(&path)
|
||||
};
|
||||
#[cfg(test)]
|
||||
super::local::volume_mutation_test_hooks::run(super::local::volume_mutation_test_hooks::Stage::CreateComplete, &path);
|
||||
result
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// Create a directory and all its parent components if they are missing.
|
||||
/// Without recursion support, fall back to create_dir_all
|
||||
/// This function will not create directories under base_dir.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) {
|
||||
async fn os_mkdir_all(dir_path: &Path, base_dir: &Path, owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
if !base_dir.to_string_lossy().is_empty() && base_dir.starts_with(dir_path) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Err(e) = super::fs::mkdir(dir_path.as_ref()).await {
|
||||
if let Err(e) = mkdir_with_owner(dir_path, false, owner.clone()).await {
|
||||
if e.kind() == io::ErrorKind::AlreadyExists {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -5656,16 +6027,16 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if let Some(parent) = dir_path.as_ref().parent() {
|
||||
if let Some(parent) = dir_path.parent() {
|
||||
// Fall back to creating the missing parent chain only when the direct mkdir proves it is required.
|
||||
if let Err(parent_err) = super::fs::make_dir_all(parent).await
|
||||
if let Err(parent_err) = mkdir_with_owner(parent, true, owner.clone()).await
|
||||
&& parent_err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(parent_err);
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(retry_err) = super::fs::mkdir(dir_path.as_ref()).await
|
||||
if let Err(retry_err) = mkdir_with_owner(dir_path, false, owner.clone()).await
|
||||
&& retry_err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(retry_err);
|
||||
@@ -5712,6 +6083,84 @@ mod tests {
|
||||
use tempfile::tempdir;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
#[tokio::test]
|
||||
async fn physical_rename_barrier_allows_control_work_without_releasing_cancelled_owner() {
|
||||
struct ReleaseBarrier(PathBuf);
|
||||
|
||||
impl Drop for ReleaseBarrier {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::write(self.0.with_extension("release"), b"release physical worker");
|
||||
}
|
||||
}
|
||||
|
||||
let temp = tempdir().expect("physical barrier fixture");
|
||||
let root = temp.path().canonicalize().expect("canonical disk root");
|
||||
let queue = register_disk_blocking_queue(&root, &root);
|
||||
let source = root.join("source");
|
||||
let destination = root.join("destination");
|
||||
std::fs::write(&source, b"physical payload").expect("staged payload");
|
||||
let metadata = root.join("control-metadata");
|
||||
std::fs::write(&metadata, b"control").expect("existing control metadata");
|
||||
let barrier = root.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
std::fs::create_dir_all(barrier.parent().expect("barrier parent")).expect("barrier directory");
|
||||
std::fs::write(&barrier, destination.to_str().expect("destination path")).expect("arm physical barrier");
|
||||
let release = ReleaseBarrier(barrier.clone());
|
||||
let lease = acquire_namespace_mutation_lease(&destination).await;
|
||||
assert!(
|
||||
Arc::ptr_eq(&lease.blocking_queue, &queue),
|
||||
"control and physical work share the disk queue"
|
||||
);
|
||||
let owner = Arc::downgrade(&lease);
|
||||
let publication_root = PublicationRoot::new(&root).expect("pinned disk root");
|
||||
let worker_destination = destination.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
reliable_rename_inner_with_lease(source, worker_destination, root, publication_root, false, lease).await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !barrier.with_extension("arrived").exists() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical worker enters the real E2E barrier");
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("cancel async rename waiter").is_cancelled());
|
||||
assert!(barrier.with_extension("waiter-dropped").exists(), "receiver observes waiter cancellation");
|
||||
|
||||
let control_owner = owner.clone();
|
||||
let control_destination = destination.clone();
|
||||
let control = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
queue.run(move || {
|
||||
Ok((
|
||||
std::fs::read(metadata)?,
|
||||
control_owner.upgrade().is_some(),
|
||||
disk_namespace_mutation_lock(&control_destination).try_write_owned().is_err(),
|
||||
control_destination.exists(),
|
||||
))
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
// Drain even when the progress oracle fails, so a regression does not strand a worker.
|
||||
drop(release);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while owner.upgrade().is_some() || !destination.exists() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical completion releases the namespace owner");
|
||||
let (bytes, retained, locked, published) = control
|
||||
.expect("control work progresses during the pause")
|
||||
.expect("control read");
|
||||
assert_eq!(bytes, b"control");
|
||||
assert!(retained && locked, "cancelled waiter must leave the physical namespace lease owned");
|
||||
assert!(!published, "control work must run before physical publication");
|
||||
assert!(disk_namespace_mutation_lock(&destination).try_write_owned().is_ok());
|
||||
assert_eq!(std::fs::read(destination).expect("completed physical publication"), b"physical payload");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_staging_leases_are_ordered_and_deduplicated() {
|
||||
let root = tempdir().expect("namespace fixture");
|
||||
@@ -5727,7 +6176,7 @@ mod tests {
|
||||
);
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while disk_namespace_mutation_lock(&early).try_lock_owned().is_ok() {
|
||||
while disk_namespace_mutation_lock(&early).try_write_owned().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
@@ -9370,6 +9819,7 @@ mod tests {
|
||||
sync,
|
||||
Some("receipt"),
|
||||
None,
|
||||
None,
|
||||
));
|
||||
assert!(futures::FutureExt::now_or_never(&mut cleanup).is_none());
|
||||
let foreground = tokio::time::timeout(Duration::from_secs(5), queue.run(|| Ok(()))).await;
|
||||
@@ -9384,6 +9834,56 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn prepared_cleanup_reuses_lease_and_retains_it_after_cancellation() {
|
||||
let root = tempdir().expect("disk root");
|
||||
let intermediate = root.path().join("intermediate");
|
||||
let receipt = root.path().join("receipt");
|
||||
std::fs::write(&intermediate, b"backup").expect("backup");
|
||||
std::fs::write(&receipt, b"committed").expect("receipt");
|
||||
let prepared = acquire_namespace_mutation_lease(&intermediate).await;
|
||||
let probe = Arc::downgrade(&prepared);
|
||||
let (entered_tx, entered_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = prepared_publication_test_hooks::install_at(
|
||||
prepared_publication_test_hooks::Stage::Remove,
|
||||
&intermediate,
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let dir = root.path().to_path_buf();
|
||||
let worker = tokio::spawn(async move {
|
||||
remove_files_and_fsync_with_owner(&dir, &["intermediate"], true, Some("receipt"), None, Some(prepared)).await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("prepared cleanup must reuse its existing key instead of waiting for itself")
|
||||
.expect("physical cleanup entered");
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("caller cancelled").is_cancelled());
|
||||
let retained = probe.upgrade().is_some();
|
||||
let locked = disk_namespace_mutation_lock(&intermediate).try_write_owned().is_err();
|
||||
release_tx.send(()).expect("release physical cleanup");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while probe.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical cleanup must release its owner");
|
||||
assert!(
|
||||
retained && locked,
|
||||
"prepared cleanup must keep its complete guard set after caller cancellation"
|
||||
);
|
||||
assert!(
|
||||
!intermediate.exists() && !receipt.exists(),
|
||||
"cleanup must remove intermediates before retiring the receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_blocking_queue_rejected_worker_releases_jobs_and_resources() {
|
||||
use crate::disk::io_admission::with_background_resource_unit;
|
||||
@@ -9733,15 +10233,16 @@ mod tests {
|
||||
let file_c = temp_dir.path().join("c");
|
||||
std::fs::write(&file_a, b"a").expect("write a");
|
||||
std::fs::write(&file_c, b"c").expect("write c");
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["a", "missing-b"], true, Some("c"), None)
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["a", "missing-b"], true, Some("c"), None, None)
|
||||
.await
|
||||
.expect("remove_files_and_fsync_with_owner must tolerate missing intermediates and remove final");
|
||||
assert!(!file_a.exists() && !file_c.exists());
|
||||
|
||||
// Missing final_remove must propagate NotFound.
|
||||
let missing_final = remove_files_and_fsync_with_owner(temp_dir.path(), &["missing-a"], false, Some("missing-c"), None)
|
||||
.await
|
||||
.expect_err("missing final file must return an error");
|
||||
let missing_final =
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["missing-a"], false, Some("missing-c"), None, None)
|
||||
.await
|
||||
.expect_err("missing final file must return an error");
|
||||
assert_eq!(missing_final.kind(), io::ErrorKind::NotFound);
|
||||
|
||||
// Non-NotFound failure on an intermediate entry must abort before final_remove.
|
||||
@@ -9750,7 +10251,7 @@ mod tests {
|
||||
std::fs::create_dir(&blocker_dir).expect("create directory blocker");
|
||||
std::fs::write(&preserved_final, b"keep").expect("write preserved final");
|
||||
assert!(
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["blocker"], true, Some("preserved-final"), None)
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["blocker"], true, Some("preserved-final"), None, None)
|
||||
.await
|
||||
.is_err(),
|
||||
"directory in place of file must fail intermediate remove_file"
|
||||
|
||||
@@ -153,10 +153,13 @@ async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters()
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep
|
||||
/// them alive for the test's duration and the directories are removed on drop.
|
||||
pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
|
||||
make_local_set_disks_with_ctx(drive_count, parity_count, crate::runtime::instance::bootstrap_ctx()).await
|
||||
/// Keep the real metadata owner and backing directories alive together.
|
||||
pub(crate) async fn make_local_set_disks(
|
||||
drive_count: usize,
|
||||
parity_count: usize,
|
||||
) -> (crate::set_disk::HermeticSetFixture, Arc<SetDisks>) {
|
||||
let (fixture, _, set) = crate::set_disk::hermetic_set_disks_for_pool_with_default_parity(drive_count, 0, parity_count).await;
|
||||
(fixture, set)
|
||||
}
|
||||
|
||||
/// Like [`make_local_set_disks`], but binds the set to an explicit instance
|
||||
@@ -756,6 +759,9 @@ async fn blackbox_range_read_restores_exact_slice_with_one_offline_disk() {
|
||||
#[tokio::test]
|
||||
async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_metadata() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = crate::bucket::metadata_sys::object_store_in(set_disks.instance_ctx())
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bb-delete-marker-read-negative";
|
||||
let object = "object.bin";
|
||||
let opts = ObjectOptions {
|
||||
@@ -765,13 +771,26 @@ async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"body hidden by delete marker".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("object should be written before delete marker");
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ mod storage_api_contracts;
|
||||
mod store;
|
||||
|
||||
pub use disk::{with_background_disk_io, with_background_resource_unit};
|
||||
pub use store::PoolMetaWriteGateStatus;
|
||||
pub use store::{BucketIncarnationFenceGuard, PoolMetaWriteGateStatus};
|
||||
|
||||
// pub mod checksum;
|
||||
mod event;
|
||||
|
||||
@@ -152,7 +152,7 @@ pub struct ObjectLockConfigSnapshot {
|
||||
config_revision: Option<OffsetDateTime>,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
_lifecycle_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
}
|
||||
|
||||
@@ -210,11 +210,36 @@ impl ObjectLockConfigSnapshot {
|
||||
config_revision: Some(config_revision),
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
_lifecycle_guard: Some(Arc::new(lifecycle_guard)),
|
||||
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_store_bucket_under_lifecycle_owner(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
config_revision: OffsetDateTime,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Self {
|
||||
let mut lifecycle_fence = NamespaceLockFence::new();
|
||||
if let Some(signal) = lifecycle_guard.lock_lost_signal() {
|
||||
lifecycle_fence.add_signal(signal);
|
||||
}
|
||||
Self {
|
||||
store_id: Some(store_id),
|
||||
bucket: Some(bucket.to_owned()),
|
||||
bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
config_revision: Some(config_revision),
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
metadata_transaction_guard: Some(metadata_transaction_guard),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_store_bucket_under_lifecycle_fence(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
@@ -254,6 +279,15 @@ impl ObjectLockConfigSnapshot {
|
||||
&self.state
|
||||
}
|
||||
|
||||
pub(crate) fn is_unbound(&self) -> bool {
|
||||
self.store_id.is_none()
|
||||
&& self.bucket.is_none()
|
||||
&& self.bucket_incarnation_id.is_none()
|
||||
&& self.config_revision.is_none()
|
||||
&& self._lifecycle_guard.is_none()
|
||||
&& self.metadata_transaction_guard.is_none()
|
||||
}
|
||||
|
||||
pub(crate) fn is_valid_for_destructive_put(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
|
||||
self.store_id == Some(store_id)
|
||||
&& self.bucket.as_deref() == Some(bucket)
|
||||
@@ -266,6 +300,11 @@ impl ObjectLockConfigSnapshot {
|
||||
.is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
pub(crate) fn owns_bucket_lifecycle_fence(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
|
||||
self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
|
||||
&& self._lifecycle_guard.as_ref().is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
/// Share the held transaction lock without queuing another reader behind
|
||||
/// a metadata writer that is itself waiting for this snapshot to drop.
|
||||
pub(crate) fn metadata_transaction_guard_for(
|
||||
@@ -962,6 +1001,12 @@ pub struct ObjectOptions {
|
||||
/// A genuine target write guard; loss fences alone do not authorize a borrowed write.
|
||||
#[doc(hidden)]
|
||||
pub write_lock_context: Option<WriteLockContext>,
|
||||
/// Ordinary ECStore PUT target bound to its bucket's immutable artifact namespace.
|
||||
#[doc(hidden)]
|
||||
pub artifact_put_target: Option<Arc<crate::bucket::artifact::ArtifactPutTarget>>,
|
||||
/// Owned, non-serializable authority for one fully sealed managed version.
|
||||
#[doc(hidden)]
|
||||
pub artifact_delete_target: Option<Arc<crate::bucket::artifact::ArtifactDeleteTarget>>,
|
||||
/// In-process delete ownership for late filesystem completion. Never
|
||||
/// serialized or copied into object metadata.
|
||||
#[doc(hidden)]
|
||||
@@ -1041,6 +1086,9 @@ pub struct ObjectOptions {
|
||||
/// Proves an upper layer holds the bucket lifecycle sentinel. A separate
|
||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||
/// Actual generation owner retained by storage commits and their physical tails.
|
||||
#[doc(hidden)]
|
||||
pub bucket_incarnation_fence: Option<Arc<crate::store::BucketIncarnationFenceGuard>>,
|
||||
/// Retains the actual lifecycle guard for an owned multipart commit. A
|
||||
/// lock-loss fence alone does not keep a cancelled caller's lock alive. The
|
||||
/// owner also retains the continuous incarnation-validation coverage.
|
||||
|
||||
@@ -71,7 +71,7 @@ impl WriteCommitGuard {
|
||||
|
||||
/// Namespace locks key the disk identity of the name (see `new_ns_lock`),
|
||||
/// so a guard proves every spelling that resolves to the same file.
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
pub(crate) fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
self.bucket.as_ref() == bucket && self.object.as_ref() == object_name_disk_identity(object).as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -814,6 +814,14 @@ impl InstanceContext {
|
||||
self.tier_config_mgr.get_or_init(TierConfigMgr::new).clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_test_tier_config_mgr(&self, manager: Arc<RwLock<TierConfigMgr>>) {
|
||||
assert!(
|
||||
self.tier_config_mgr.set(manager).is_ok(),
|
||||
"test tier manager must be installed before use"
|
||||
);
|
||||
}
|
||||
|
||||
/// This instance's event notifier, materializing it on first access.
|
||||
/// Shared for the lifetime of the instance.
|
||||
pub fn event_notifier(&self) -> Arc<RwLock<EventNotifier>> {
|
||||
|
||||
@@ -250,7 +250,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let bucket_incarnation_fence = match bucket_configs.bucket_incarnation_id {
|
||||
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||
Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
|
||||
None => None,
|
||||
};
|
||||
|
||||
@@ -533,9 +533,8 @@ impl ECStore {
|
||||
&cleanup_transferred_tier_versions,
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: bucket_configs.bucket_incarnation_id,
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
lifecycle_owner: bucket_incarnation_fence.as_ref(),
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: lock_lost_signal.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -1729,6 +1728,8 @@ mod tests {
|
||||
/// Both versions are on the target with their identities, the null
|
||||
/// version's content and SHA-256 are intact, and the source is gone.
|
||||
async fn assert_moved(&self) {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let target = self.store.pools[1]
|
||||
.get_disks_by_key(self.object)
|
||||
.load_file_info_versions_exact(self.bucket, self.object)
|
||||
@@ -1769,6 +1770,16 @@ mod tests {
|
||||
self.source_versions().await.is_none(),
|
||||
"the source entry must be removed only after both versions moved"
|
||||
);
|
||||
let source_disks = self.source_set.disk_inventory().await;
|
||||
assert_eq!(source_disks.iter().flatten().count(), 2, "both source disks must be online");
|
||||
for disk in source_disks.into_iter().flatten() {
|
||||
assert!(
|
||||
!tokio::fs::try_exists(disk.path().join(self.bucket).join(self.object))
|
||||
.await
|
||||
.expect("inspect the physical rebalance source path"),
|
||||
"rebalance must remove source metadata and shards from every disk"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -88,6 +88,7 @@ pub async fn test_store_with_persisted_rebalance_meta(
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
(temp_dirs, store)
|
||||
}
|
||||
@@ -266,6 +267,7 @@ async fn test_pool_stores_with_contexts(
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
|
||||
ctx: store_ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
};
|
||||
let store = make_store(ctx);
|
||||
|
||||
@@ -3415,6 +3415,7 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -3053,7 +3053,7 @@ impl RenameRollbackReceipt {
|
||||
|
||||
struct RenameRollbackOwnership {
|
||||
receipt: Option<RenameRollbackReceipt>,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
async fn inspect_incomplete_rename_rollback(
|
||||
@@ -3169,7 +3169,7 @@ async fn rollback_failed_rename(
|
||||
old_data_dir: rollback_dir,
|
||||
..Default::default()
|
||||
},
|
||||
disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
disk_namespace_commit_guard,
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -3281,10 +3281,11 @@ async fn record_rename_rollback_outcomes(
|
||||
/// quorum and optional scanner lease map together avoids widening either
|
||||
/// fanout helper's argument list while preserving the fence semantics.
|
||||
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
write_quorum: usize,
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
}
|
||||
|
||||
@@ -3294,6 +3295,7 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
artifact_put: None,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope: None,
|
||||
@@ -3302,6 +3304,14 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_artifact_put(
|
||||
mut self,
|
||||
identity: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
) -> Self {
|
||||
self.artifact_put = identity;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self {
|
||||
self.rollback_receipt = Some(receipt);
|
||||
self
|
||||
@@ -3317,7 +3327,7 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
|
||||
pub(in crate::set_disk) fn with_namespace_commit_guard(
|
||||
mut self,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Self {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
@@ -3770,6 +3780,7 @@ impl SetDisks {
|
||||
fence_options: RenameDataFenceOptions<'_>,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
let RenameDataFenceOptions {
|
||||
artifact_put,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope: _scanner_publication_commit_scope,
|
||||
@@ -3864,11 +3875,10 @@ impl SetDisks {
|
||||
&file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
crate::disk::RenameDataGuards {
|
||||
artifact_put,
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
namespace_owner: namespace_commit_guard.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4177,6 +4187,7 @@ impl SetDisks {
|
||||
.await;
|
||||
}
|
||||
let RenameDataFenceOptions {
|
||||
artifact_put,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope,
|
||||
@@ -4297,11 +4308,10 @@ impl SetDisks {
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
crate::disk::RenameDataGuards {
|
||||
artifact_put,
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
namespace_owner: namespace_commit_guard.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -12815,6 +12825,118 @@ mod tests {
|
||||
assert!(missing.below_quorum, "missing disk slots should remain visible in the quorum lens");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn legacy_path_delete_flow_prefix_delete_keeps_scanner_owner_until_drain() {
|
||||
use crate::disk::os::{drain_namespace_for_test, prepared_publication_test_hooks as hooks};
|
||||
use crate::object_api::{
|
||||
ScannerPublicationCommitScope, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
|
||||
};
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::storage_api_contracts::bucket::BucketOperations;
|
||||
|
||||
let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "path-delete-scanner";
|
||||
crate::bucket::metadata_sys::object_store_in(&set.ctx)
|
||||
.await
|
||||
.expect("fixture object store")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("authoritative bucket creation");
|
||||
let mut paths = Vec::new();
|
||||
for disk in &disks {
|
||||
disk.write_all(bucket, "prefix/part.1", Bytes::from_static(b"old shard"))
|
||||
.await
|
||||
.expect("seed prefix on every disk");
|
||||
paths.push(
|
||||
disk.get_object_path_for_io_if_local(bucket, "prefix")
|
||||
.expect("local disk")
|
||||
.expect("prefix path"),
|
||||
);
|
||||
}
|
||||
let movement = Arc::new(tokio::sync::RwLock::new(()));
|
||||
let scope = ScannerPublicationCommitScope::new_storage_owned(
|
||||
7,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
Vec::new(),
|
||||
movement.clone().read_owned().await,
|
||||
);
|
||||
scope.try_begin().expect("begin scanner publication");
|
||||
let caller_guard = ScannerPublicationCommitScopeGuard::new(scope.clone());
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install_at(hooks::Stage::Rename, &paths[0], move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let owner_set = set.clone();
|
||||
let owner_scope = scope.clone();
|
||||
let caller = tokio::spawn(async move {
|
||||
owner_set
|
||||
.delete_prefix_with_scanner_publication_lease(bucket, "prefix", None, Some(owner_scope))
|
||||
.await
|
||||
});
|
||||
let entered = tokio::time::timeout(Duration::from_secs(5), entered_rx).await;
|
||||
caller.abort();
|
||||
let _ = caller.await;
|
||||
drop(caller_guard);
|
||||
let state_while_blocked = scope.state();
|
||||
let movement_while_blocked = movement.clone().try_write_owned().is_ok();
|
||||
let disk::Disk::Local(local) = disks[0].as_ref() else {
|
||||
unreachable!("fixture contains local disks");
|
||||
};
|
||||
let local = local.get_disk();
|
||||
let volume_read_held = crate::disk::os::disk_volume_mutation_lock(&local.root, bucket)
|
||||
.try_write_owned()
|
||||
.is_err();
|
||||
let mut deletion = Box::pin(local.delete_volume(bucket, true));
|
||||
let early_deletion = futures::poll!(deletion.as_mut());
|
||||
let deletion_was_blocked = early_deletion.is_pending();
|
||||
// Always release and drain physical work before reporting an ownership failure.
|
||||
let _ = release_tx.send(());
|
||||
let mut drained = Vec::new();
|
||||
for path in &paths {
|
||||
drained.push(tokio::time::timeout(Duration::from_secs(5), drain_namespace_for_test(path)).await);
|
||||
}
|
||||
let deletion_result = match early_deletion {
|
||||
std::task::Poll::Ready(result) => result,
|
||||
std::task::Poll::Pending => tokio::time::timeout(Duration::from_secs(5), deletion)
|
||||
.await
|
||||
.expect("volume deletion drains"),
|
||||
};
|
||||
let terminal = tokio::time::timeout(Duration::from_secs(5), scope.wait_for_completion()).await;
|
||||
assert!(
|
||||
entered.is_ok_and(|result| result.is_ok()),
|
||||
"real prefix deletion must reach its filesystem worker"
|
||||
);
|
||||
assert!(drained.into_iter().all(|result| result.is_ok()), "every physical namespace must drain");
|
||||
deletion_result.expect("volume deletion after physical drain");
|
||||
assert!(
|
||||
volume_read_held && deletion_was_blocked,
|
||||
"volume deletion must wait for the physical prefix worker"
|
||||
);
|
||||
assert_eq!(state_while_blocked, ScannerPublicationCommitState::InFlight);
|
||||
assert!(!movement_while_blocked, "scanner movement remains fenced after caller cancellation");
|
||||
assert_eq!(terminal.expect("scanner owner completion"), ScannerPublicationCommitState::Committed);
|
||||
assert!(scope.release_movement_permit().await, "terminal scanner owner releases movement");
|
||||
assert!(movement.try_write_owned().is_ok());
|
||||
for path in paths.iter().skip(1) {
|
||||
assert!(!path.exists(), "successful prefix fanout removes every surviving volume's prefix");
|
||||
}
|
||||
disks[0].make_volume(bucket).await.expect("recreate deleted volume");
|
||||
disks[0]
|
||||
.write_all(bucket, "prefix/sentinel", Bytes::from_static(b"new generation"))
|
||||
.await
|
||||
.expect("new generation sentinel");
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.read_all(bucket, "prefix/sentinel")
|
||||
.await
|
||||
.expect("sentinel survives old deletion"),
|
||||
b"new generation".as_slice()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_succeeds_when_present_disks_reach_quorum() {
|
||||
let bucket = "delete-prefix-bucket";
|
||||
|
||||
@@ -906,6 +906,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::hermetic_set_disks_support::{HermeticSetFixture, hermetic_set_disks_for_pool_with_default_parity};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
@@ -1145,6 +1147,310 @@ mod prepared_get_object_metadata_tests {
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const OBSERVATION_HOOK_BODY_LEN: usize = 1024 * 1024 + 1;
|
||||
|
||||
struct ObservationBodyHook {
|
||||
object: &'static str,
|
||||
calls: Arc<AtomicU64>,
|
||||
body: bytes::Bytes,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::object_api::GetObjectBodyCacheHook for ObservationBodyHook {
|
||||
async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<bytes::Bytes> {
|
||||
if bucket == crate::disk::RUSTFS_META_BUCKET && object == self.object {
|
||||
assert_eq!(info.size, i64::try_from(self.body.len()).expect("hook fixture size"));
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
Some(self.body.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationHookGuard;
|
||||
|
||||
impl Drop for ObservationHookGuard {
|
||||
fn drop(&mut self) {
|
||||
crate::object_api::unregister_get_object_body_cache_hook();
|
||||
}
|
||||
}
|
||||
|
||||
async fn ordinary_observation_get(set: &SetDisks, object: &str) -> Vec<u8> {
|
||||
let mut reader = set
|
||||
.get_object_reader(crate::disk::RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("ordinary GET positive control");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("ordinary GET body");
|
||||
bytes
|
||||
}
|
||||
|
||||
fn observation_register_hook(object: &'static str, body: Vec<u8>) -> (ObservationHookGuard, Arc<AtomicU64>) {
|
||||
let calls = Arc::new(AtomicU64::new(0));
|
||||
crate::object_api::register_get_object_body_cache_hook(Arc::new(ObservationBodyHook {
|
||||
object,
|
||||
calls: calls.clone(),
|
||||
body: bytes::Bytes::from(body),
|
||||
}));
|
||||
(ObservationHookGuard, calls)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn locked_observation_uses_real_outer_guard() {
|
||||
let object = "config/locked-observation-real.bin";
|
||||
let payload = vec![b'p'; OBSERVATION_HOOK_BODY_LEN];
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist locked observation fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; payload.len()]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let mut contender = Box::pin(crate::object_api::WriteCommitGuard::acquire(
|
||||
&namespace,
|
||||
std::time::Duration::from_secs(30),
|
||||
));
|
||||
assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
crate::config::com::read_config_observation_no_lock(set.clone(), object, payload.len(), &guard),
|
||||
)
|
||||
.await
|
||||
.expect("locked observation hang watchdog");
|
||||
assert!(result.is_ok(), "I06L_NO_NESTED_LOCK: {result:?}");
|
||||
let (bytes, info) = result.expect("locked observation");
|
||||
assert!(bytes == payload, "I06L_HOOK_BYPASS");
|
||||
assert!(calls.load(Ordering::SeqCst) == 0, "I06L_HOOK_BYPASS");
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.data_dir, written.data_dir);
|
||||
assert_eq!(info.name, object);
|
||||
assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
|
||||
drop(guard);
|
||||
let next = tokio::time::timeout(std::time::Duration::from_secs(10), contender)
|
||||
.await
|
||||
.expect("owner release watchdog")
|
||||
.expect("competing owner acquired");
|
||||
drop(next);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn locked_observation_preserves_empty_missing_and_bound() {
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
for (object, body) in [
|
||||
("config/locked-empty.bin", Vec::new()),
|
||||
("config/locked-bound.bin", b"raw".to_vec()),
|
||||
] {
|
||||
let mut reader = PutObjReader::from_vec(body.clone());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist boundary fixture");
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching boundary owner");
|
||||
let result = crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len(), &guard).await;
|
||||
assert!(result.is_ok(), "I06L_EMPTY_DISTINCT: {result:?}");
|
||||
let (bytes, info) = result.expect("boundary observation");
|
||||
assert_eq!(bytes, body);
|
||||
assert_eq!(info.size, written.size);
|
||||
if !body.is_empty() {
|
||||
let result =
|
||||
crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len() - 1, &guard).await;
|
||||
assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_BOUND_ENFORCED");
|
||||
}
|
||||
}
|
||||
let object = "config/locked-absent.bin";
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching missing owner");
|
||||
let result = crate::config::com::read_config_observation_no_lock(set, object, 0, &guard).await;
|
||||
assert!(matches!(result, Err(Error::ConfigNotFound)), "I06L_EMPTY_DISTINCT");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_real_backend_boundaries() {
|
||||
let object = "config/observation-real.bin";
|
||||
let payload = b"persisted config";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(payload.to_vec());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let result = crate::config::com::read_config_observation(set.clone(), object, payload.len()).await;
|
||||
assert!(result.is_ok(), "I05_OBSERVATION_SUCCESS: {result:?}");
|
||||
let (bytes, info) = result.expect("real observation success");
|
||||
assert_eq!(bytes, payload);
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.data_dir, written.data_dir);
|
||||
assert_eq!(info.name, object);
|
||||
let over = crate::config::com::read_config_observation(set.clone(), object, payload.len() - 1).await;
|
||||
assert!(over.is_err(), "I05_BOUND_ENFORCED");
|
||||
assert_eq!(over.expect_err("over limit must fail"), Error::CorruptedFormat, "I05_BOUND_ENFORCED");
|
||||
let absent = crate::config::com::read_config_observation(set.clone(), "config/observation-absent.bin", 32).await;
|
||||
assert_eq!(absent.expect_err("absent config"), Error::ConfigNotFound);
|
||||
let empty_object = "config/observation-empty.bin";
|
||||
let mut empty = PutObjReader::from_vec(Vec::new());
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, empty_object, &mut empty, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist empty config");
|
||||
let empty = crate::config::com::read_config_observation(set, empty_object, 0).await;
|
||||
assert!(empty.is_ok(), "existing empty config must succeed: {empty:?}");
|
||||
let (bytes, info) = empty.expect("empty observation");
|
||||
assert!(bytes.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_bypasses_registered_body_hook() {
|
||||
let object = "config/observation-hook.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let result = crate::config::com::read_config_observation(set, object, OBSERVATION_HOOK_BODY_LEN).await;
|
||||
assert!(result.is_ok(), "I05_HOOK_BYPASS: {result:?}");
|
||||
assert_eq!(
|
||||
result.expect("observed persisted body").0,
|
||||
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
|
||||
"I05_HOOK_BYPASS"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0, "I05_HOOK_BYPASS");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_hook_scope_restores_after_error() {
|
||||
let object = "config/observation-hook-error.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
let result = crate::config::com::read_config_observation(set.clone(), object, OBSERVATION_HOOK_BODY_LEN - 1).await;
|
||||
assert_eq!(result.expect_err("bounded observation error"), Error::CorruptedFormat);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
struct PendingObservationStore {
|
||||
set: Arc<SetDisks>,
|
||||
ready: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PendingObservationStore {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("PendingObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for PendingObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
let mut reader = self.set.get_object_reader(bucket, object, range, headers, opts).await?;
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("pending producer real GET");
|
||||
assert_eq!(
|
||||
body,
|
||||
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
|
||||
"I05_HOOK_BYPASS: producer must enter its own scope"
|
||||
);
|
||||
self.ready
|
||||
.lock()
|
||||
.expect("pending signal")
|
||||
.take()
|
||||
.expect("one GET")
|
||||
.send(())
|
||||
.expect("producer entry signal");
|
||||
std::future::pending::<Result<GetObjectReader>>().await
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
panic!("observation must not PUT");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_hook_scope_restores_after_cancel() {
|
||||
let object = "config/observation-hook-cancel.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let adapter = Arc::new(PendingObservationStore {
|
||||
set: set.clone(),
|
||||
ready: std::sync::Mutex::new(Some(ready_tx)),
|
||||
});
|
||||
let mut scope = Box::pin(crate::config::com::read_config_observation(adapter, object, OBSERVATION_HOOK_BODY_LEN));
|
||||
tokio::select! {
|
||||
result = ready_rx => { result.expect("producer reached pending GET"); }
|
||||
result = &mut scope => { panic!("producer must remain pending until cancelled: {result:?}"); }
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
drop(scope);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
|
||||
@@ -16616,7 +16922,14 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
|
||||
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-versioned-delete";
|
||||
let object = "object.txt";
|
||||
let opts = ObjectOptions {
|
||||
@@ -16625,13 +16938,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"versioned object body".to_vec());
|
||||
let written = set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned object should be written");
|
||||
assert!(written.version_id.is_some());
|
||||
@@ -16948,7 +17274,14 @@ mod tests {
|
||||
/// `BucketNotEmpty` for a bucket the client has already emptied.
|
||||
#[tokio::test]
|
||||
async fn set_level_explicit_null_version_delete_purges_the_null_delete_marker() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-null-marker-purge";
|
||||
let object = "object.txt";
|
||||
// The delete path reads versioned/suspended from the bucket-config
|
||||
@@ -16970,13 +17303,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suspended-version object should be written");
|
||||
|
||||
@@ -17001,7 +17347,20 @@ mod tests {
|
||||
|
||||
let mut replacement_reader = PutObjReader::from_vec(b"replacement suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut replacement_reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut replacement_reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a suspended-version PUT should replace the first null marker");
|
||||
let replacement_marker = set_disks
|
||||
@@ -17061,7 +17420,14 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_level_version_suspended_delete_creates_null_delete_marker() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-version-suspended-delete";
|
||||
let object = "object.txt";
|
||||
let opts = ObjectOptions {
|
||||
@@ -17071,13 +17437,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suspended-version object should be written");
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,7 +15,9 @@
|
||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
||||
use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
|
||||
use crate::set_disk::CompletePart;
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::{
|
||||
hermetic_set_disks_for_pool_with_default_parity_isolated, lifecycle_owned_put_options,
|
||||
};
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
@@ -39,18 +41,19 @@ async fn remote_metadata_heal_does_not_certify_unread_tier_payload() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
set.put_object(
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]),
|
||||
&ObjectOptions {
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("protected source");
|
||||
.await;
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]), &opts)
|
||||
.await
|
||||
.expect("protected source");
|
||||
for (disk, dir) in disks.iter().zip(&dirs) {
|
||||
let mut info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
@@ -94,10 +97,15 @@ async fn rollout_requires_both_flags_for_new_puts() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
for (requested, confirmed, protected) in [
|
||||
(false, false, false),
|
||||
@@ -159,12 +167,17 @@ async fn legacy_shard_repair_and_explicit_version_metadata_recovery_remain_avail
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let expected = vec![0x67; 1024 * 1024 + 37];
|
||||
for case in ["missing-part", "bad-bitrot", "missing-version-metadata"] {
|
||||
let object = case;
|
||||
@@ -272,10 +285,15 @@ async fn multipart_mode_survives_rollout_switch_changes() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
for protected in [false, true] {
|
||||
let object = format!("upload-{protected}");
|
||||
let upload =
|
||||
@@ -354,11 +372,16 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
|
||||
};
|
||||
let object = format!("source-{protected}");
|
||||
let expected = vec![0x39; 1024 * 1024 + 11];
|
||||
let seed_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
..Default::default()
|
||||
};
|
||||
let seed_opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let mut source = set
|
||||
.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &seed_opts)
|
||||
.await
|
||||
@@ -366,8 +389,8 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
|
||||
source.metadata_only = false;
|
||||
source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
shard_integrity_write_mode: None,
|
||||
..seed_opts.clone()
|
||||
};
|
||||
let copied = temp_env::async_with_vars(
|
||||
rollout_vars(!protected, !protected),
|
||||
|
||||
@@ -2247,7 +2247,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// lifetime; a fence by itself cannot transfer the outer lock lease.
|
||||
let detach_commit_owner = _upload_commit_guard.is_some()
|
||||
&& opts.namespace_lock_fence.is_none()
|
||||
&& (opts.bucket_lifecycle_lock_fence.is_none() || opts.bucket_lifecycle_lock_owner.is_some());
|
||||
&& (opts.bucket_lifecycle_lock_fence.is_none()
|
||||
|| opts.bucket_lifecycle_lock_owner.is_some()
|
||||
|| opts.bucket_incarnation_fence.is_some());
|
||||
let commit_set = self.clone();
|
||||
let commit_bucket = bucket.to_owned();
|
||||
let commit_object = object.to_owned();
|
||||
@@ -3902,7 +3904,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(write_quorum, None)
|
||||
.with_namespace_commit_guard(
|
||||
(!crate::bucket::utils::is_meta_bucketname(&commit_bucket))
|
||||
.then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
.then(|| commit_set.ctx.begin_namespace_commit() as Arc<dyn Send + Sync>),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
@@ -4179,14 +4181,12 @@ mod tests {
|
||||
use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test;
|
||||
use crate::set_disk::core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
// No-locker helpers resolve to the isolated-context variants (see
|
||||
// `hermetic_set_disks_isolated`); the guard-based tests build through
|
||||
// `hermetic_set_disks_with_lockers`, which stays on the bootstrap context
|
||||
// so their own ambient SetupTypeGuard flip remains visible to them.
|
||||
// All fixtures own isolated metadata contexts; lock-failure tests switch
|
||||
// the same context to distributed mode after seeding their objects.
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::{
|
||||
hermetic_set_disks_for_pool_with_default_parity_isolated as hermetic_set_disks_for_pool_with_default_parity,
|
||||
hermetic_set_disks_isolated as hermetic_set_disks, hermetic_set_disks_with_lockers,
|
||||
hermetic_set_disks_with_lockers_and_ctx,
|
||||
hermetic_set_disks_with_lockers_and_ctx, lifecycle_owned_put_options,
|
||||
};
|
||||
use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
@@ -4408,24 +4408,33 @@ mod tests {
|
||||
}
|
||||
|
||||
struct SetupTypeGuard {
|
||||
ctx: Arc<crate::runtime::instance::InstanceContext>,
|
||||
previous: SetupType,
|
||||
}
|
||||
|
||||
impl SetupTypeGuard {
|
||||
async fn switch_to(next: SetupType) -> Self {
|
||||
let previous = crate::runtime::sources::current_setup_type().await;
|
||||
crate::runtime::sources::set_setup_type(next).await;
|
||||
Self { previous }
|
||||
async fn switch_to(ctx: &Arc<crate::runtime::instance::InstanceContext>, next: SetupType) -> Self {
|
||||
let previous = if ctx.is_dist_erasure().await {
|
||||
SetupType::DistErasure
|
||||
} else {
|
||||
SetupType::Erasure
|
||||
};
|
||||
ctx.update_erasure_type(next).await;
|
||||
Self {
|
||||
ctx: Arc::clone(ctx),
|
||||
previous,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SetupTypeGuard {
|
||||
fn drop(&mut self) {
|
||||
let previous = self.previous.clone();
|
||||
let ctx = Arc::clone(&self.ctx);
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
std::thread::spawn(move || {
|
||||
handle.block_on(async {
|
||||
crate::runtime::sources::set_setup_type(previous).await;
|
||||
ctx.update_erasure_type(previous).await;
|
||||
});
|
||||
})
|
||||
.join()
|
||||
@@ -5397,10 +5406,15 @@ mod tests {
|
||||
bucket,
|
||||
object,
|
||||
&mut initial_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
&lifecycle_owned_put_options(
|
||||
&set_disks,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
.expect("initial fenced PUT should commit");
|
||||
@@ -5439,10 +5453,15 @@ mod tests {
|
||||
bucket,
|
||||
object,
|
||||
&mut winner_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
&lifecycle_owned_put_options(
|
||||
&set_disks,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
.expect("concurrent fenced PUT should advance the epoch");
|
||||
@@ -6185,15 +6204,22 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x47; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
@@ -6236,15 +6262,22 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn assert_abort_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x48; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let object_holder = set_disks
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
@@ -6300,17 +6333,365 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn assert_automatic_part_rollback(settlement_timeout: bool) {
|
||||
use crate::disk::{
|
||||
Disk,
|
||||
disk_store::LocalDiskWrapper,
|
||||
health_state::RuntimeDriveHealthState,
|
||||
os::{drain_namespace_for_test, prepared_publication_test_hooks as hooks},
|
||||
};
|
||||
use crate::io_support::shard_integrity::IntegrityBuilder;
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
use rustfs_filemeta::shard_integrity::IntegrityLayout;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("5")),
|
||||
(crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
|
||||
],
|
||||
async {
|
||||
for old in [false, true] {
|
||||
let (_roots, original_disks, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
|
||||
// This fixture normally disables health monitors. Install ordinary
|
||||
// monitored wrappers before any scenario I/O, on the same four disks.
|
||||
let disks = original_disks
|
||||
.iter()
|
||||
.map(|disk| match disk.as_ref() {
|
||||
Disk::Local(local) => Arc::new(Disk::Local(Box::new(LocalDiskWrapper::new(local.get_disk(), true)))),
|
||||
Disk::Remote(_) => unreachable!("four local disks"),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
*set_disks.disks.write().await = disks.iter().cloned().map(Some).collect();
|
||||
assert!(disks.iter().all(|disk| disk.health_check_enabled_for_test()));
|
||||
let bucket = "part-tail-rollback";
|
||||
let object = "object";
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
let (destination, completion) = if old {
|
||||
let payload = vec![0x41; 1 << 20];
|
||||
let create_opts = ObjectOptions {
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &create_opts).await;
|
||||
let (info, _) = set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload_id, false)
|
||||
.await
|
||||
.expect("acknowledged upload");
|
||||
(
|
||||
format!(
|
||||
"{}/{}/part.1",
|
||||
SetDisks::get_upload_id_dir(bucket, object, &upload_id),
|
||||
info.data_dir.expect("data directory")
|
||||
),
|
||||
Some((upload_id, parts, payload)),
|
||||
)
|
||||
} else {
|
||||
(format!("{}/part.1", Uuid::new_v4()), None)
|
||||
};
|
||||
let source = format!("{}/part.1", Uuid::new_v4());
|
||||
let mut builder =
|
||||
IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("index builder");
|
||||
builder.push([b"bbbb".as_slice(); 4].into_iter()).await.expect("stripe");
|
||||
let proof = builder.finish(8).await.expect("proof");
|
||||
let index = proof.inline_bytes().expect("inline index");
|
||||
let proof_name = proof.part.file_name();
|
||||
let meta = Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 8,
|
||||
integrity: Some(proof.part),
|
||||
etag: "tail-retry".to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("metadata"),
|
||||
);
|
||||
let mut snapshots = Vec::new();
|
||||
let mut paths = Vec::new();
|
||||
for disk in &disks {
|
||||
let target = disk
|
||||
.get_object_path_for_io_if_local(RUSTFS_META_MULTIPART_BUCKET, &destination)
|
||||
.expect("local disk")
|
||||
.expect("target");
|
||||
let old_data = std::fs::read(&target).ok();
|
||||
let old_meta = std::fs::read(target.with_extension("1.meta")).ok();
|
||||
let old_index = old_meta
|
||||
.as_ref()
|
||||
.and_then(|bytes| ObjectPartInfo::unmarshal(bytes).ok())
|
||||
.and_then(|part| part.integrity)
|
||||
.map(|integrity| {
|
||||
let path = target.parent().expect("parent").join(integrity.file_name());
|
||||
let bytes = std::fs::read(&path).expect("acknowledged index");
|
||||
(path, bytes)
|
||||
});
|
||||
if old {
|
||||
assert!(old_index.is_some(), "acknowledged protected upload must have an integrity index");
|
||||
}
|
||||
snapshots.push((old_data, old_meta, old_index));
|
||||
paths.push(target);
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, &source, Bytes::from_static(b"bbbb"))
|
||||
.await
|
||||
.expect("retry shard");
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&format!("{}/{}", source.rsplit_once('/').expect("directory").0, proof_name),
|
||||
index.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("retry index");
|
||||
}
|
||||
let mut guards = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut entered = Vec::new();
|
||||
let mut settle_waiting = Vec::new();
|
||||
let mut settle_completed = Vec::new();
|
||||
let mut settle_timed_out = Vec::new();
|
||||
let mut restored = Vec::new();
|
||||
let mut finished = Vec::new();
|
||||
// Two of four publications cannot acknowledge the required quorum of three.
|
||||
for (number, target) in paths.iter().take(2).enumerate() {
|
||||
let paused_path = if number == 0 {
|
||||
target.clone()
|
||||
} else {
|
||||
target.with_extension("1.meta")
|
||||
};
|
||||
let transaction = target.parent().expect("parent").join(".part.1.rustfs-txn");
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let (release, blocked) = std::sync::mpsc::channel::<()>();
|
||||
guards.push(hooks::install_at(hooks::Stage::Rename, &paused_path, move || {
|
||||
let _ = tx.send(());
|
||||
let _ = blocked.recv();
|
||||
}));
|
||||
entered.push(rx);
|
||||
releases.push(release);
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
guards.push(hooks::install_at(hooks::Stage::NamespaceWaiting, target, move || {
|
||||
let _ = tx.send(());
|
||||
}));
|
||||
settle_waiting.push(rx);
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
guards.push(hooks::install_at(hooks::Stage::PartSettleTimeout, target, move || {
|
||||
let _ = tx.send(());
|
||||
}));
|
||||
settle_timed_out.push(rx);
|
||||
let restore = Arc::new(AtomicBool::new(false));
|
||||
let observed = restore.clone();
|
||||
guards.push(hooks::install_at(hooks::Stage::PartRestore, &transaction, move || {
|
||||
observed.store(true, Ordering::SeqCst);
|
||||
}));
|
||||
restored.push(restore);
|
||||
let complete = Arc::new(AtomicBool::new(false));
|
||||
let observed = complete.clone();
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
guards.push(hooks::install_at(hooks::Stage::PartSettleComplete, target, move || {
|
||||
observed.store(true, Ordering::SeqCst);
|
||||
let _ = tx.send(());
|
||||
}));
|
||||
settle_completed.push(rx);
|
||||
finished.push(complete);
|
||||
}
|
||||
let operation_set = set_disks.clone();
|
||||
let operation_disks = disks.iter().cloned().map(Some).collect::<Vec<_>>();
|
||||
let operation_source = source.clone();
|
||||
let operation_destination = destination.clone();
|
||||
let operation = tokio::spawn(async move {
|
||||
operation_set
|
||||
.rename_part(
|
||||
&operation_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&operation_source,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&operation_destination,
|
||||
meta,
|
||||
3,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
});
|
||||
let mut waited = Vec::new();
|
||||
let mut before_drain = Vec::new();
|
||||
let mut after_timeout = Vec::new();
|
||||
// Keep observations until both physical workers have drained. Even a
|
||||
// deliberately broken admission path must not strand a test worker.
|
||||
let observed = async {
|
||||
for receiver in entered {
|
||||
timeout(Duration::from_secs(30), receiver)
|
||||
.await
|
||||
.map_err(|_| "publisher did not reach barrier")?
|
||||
.map_err(|_| "publisher hook dropped")?;
|
||||
}
|
||||
for (waiting, completed) in settle_waiting.into_iter().zip(settle_completed) {
|
||||
let pending = timeout(Duration::from_secs(15), async {
|
||||
tokio::select! {
|
||||
result = waiting => result.map(|()| true),
|
||||
result = completed => result.map(|()| false),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| "automatic settlement neither waited nor completed")?
|
||||
.map_err(|_| "settlement observation dropped")?;
|
||||
waited.push(pending);
|
||||
}
|
||||
for (number, target) in paths.iter().take(2).enumerate() {
|
||||
let transaction = target.parent().expect("parent").join(".part.1.rustfs-txn");
|
||||
before_drain.push((
|
||||
restored[number].load(Ordering::SeqCst),
|
||||
finished[number].load(Ordering::SeqCst),
|
||||
transaction.join("new.meta").exists(),
|
||||
transaction.join("rollback").exists(),
|
||||
std::fs::read(target.with_extension("1.meta")).ok(),
|
||||
));
|
||||
}
|
||||
if settlement_timeout && waited.iter().all(|pending| *pending) {
|
||||
for receiver in settle_timed_out {
|
||||
timeout(Duration::from_secs(15), receiver)
|
||||
.await
|
||||
.map_err(|_| "settlement did not report Timeout")?
|
||||
.map_err(|_| "timeout observation dropped")?;
|
||||
}
|
||||
for (number, target) in paths.iter().take(2).enumerate() {
|
||||
let transaction = target.parent().expect("parent").join(".part.1.rustfs-txn");
|
||||
after_timeout.push((
|
||||
transaction.join("new.meta").exists(),
|
||||
transaction.join("rollback").exists(),
|
||||
restored[number].load(Ordering::SeqCst),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok::<(), &str>(())
|
||||
}
|
||||
.await;
|
||||
drop(releases);
|
||||
let mut drained = Vec::new();
|
||||
for target in paths.iter().take(2) {
|
||||
drained.push(timeout(Duration::from_secs(15), drain_namespace_for_test(target)).await);
|
||||
}
|
||||
let result = timeout(Duration::from_secs(30), operation).await;
|
||||
for result in drained {
|
||||
result.expect("actual publisher namespace drains");
|
||||
}
|
||||
observed.expect("observe automatic rollback under real deadlines");
|
||||
assert_eq!(waited, [true, true], "automatic rollback must really wait for the publisher's D lock");
|
||||
for (number, (restored, finished, has_meta, has_rollback, meta)) in before_drain.iter().enumerate() {
|
||||
assert!(!restored, "automatic rollback must not restore before the old publisher drains");
|
||||
assert!(!finished, "automatic rollback must wait for D, not return a competing-namespace error");
|
||||
assert!(*has_meta);
|
||||
assert!(!has_rollback, "no rollback write precedes physical drain");
|
||||
assert_eq!(meta, &snapshots[number].1);
|
||||
}
|
||||
if settlement_timeout {
|
||||
assert_eq!(after_timeout.len(), 2);
|
||||
for (has_meta, has_rollback, restored) in after_timeout {
|
||||
assert!(has_meta, "timed-out settlement retains recovery input");
|
||||
assert!(!has_rollback && !restored);
|
||||
}
|
||||
}
|
||||
let result = result.expect("automatic recovery returns").expect("quorum task");
|
||||
assert!(
|
||||
matches!(result, Err(DiskError::ErasureWriteQuorum)),
|
||||
"publication preserves its original quorum failure: {result:?}"
|
||||
);
|
||||
if settlement_timeout {
|
||||
// Two consecutive real timeouts take these drives offline. Only
|
||||
// production health probes may make them available to recovery.
|
||||
timeout(Duration::from_secs(30), async {
|
||||
let mut poll = tokio::time::interval(Duration::from_millis(25));
|
||||
while disks
|
||||
.iter()
|
||||
.any(|disk| disk.runtime_state() != RuntimeDriveHealthState::Online)
|
||||
{
|
||||
poll.tick().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("real recovery monitors restore all four disks");
|
||||
let directory = format!("{}/", destination.rsplit_once('/').expect("directory").0);
|
||||
set_disks
|
||||
.recover_part_transactions(&directory, 2, 3)
|
||||
.await
|
||||
.expect("existing recovery converges after monitor recovery");
|
||||
}
|
||||
// The ordinary case reaches these assertions directly from the
|
||||
// real rename_part return, with no supplemental recovery call.
|
||||
for (number, target) in paths.iter().enumerate() {
|
||||
assert_eq!(std::fs::read(target).ok(), snapshots[number].0, "data on disk {number}");
|
||||
assert_eq!(
|
||||
std::fs::read(target.with_extension("1.meta")).ok(),
|
||||
snapshots[number].1,
|
||||
"metadata on disk {number}"
|
||||
);
|
||||
if let Some((path, bytes)) = &snapshots[number].2 {
|
||||
assert_eq!(&std::fs::read(path).expect("old index survives"), bytes);
|
||||
}
|
||||
assert!(!target.parent().expect("parent").join(".part.1.rustfs-txn").exists());
|
||||
}
|
||||
if let Some((upload_id, parts, payload)) = completion {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("acknowledged old upload stays completable");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read restored object");
|
||||
assert_eq!(bytes, payload);
|
||||
}
|
||||
drop(guards);
|
||||
for disk in &disks {
|
||||
if let Disk::Local(local) = disk.as_ref() {
|
||||
local.stop_monitoring().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn legacy_part_mutation_flow_automatic_quorum_rollback_waits_for_tail() {
|
||||
assert_automatic_part_rollback(false).await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn legacy_part_mutation_flow_settlement_timeout_retains_recoverable_state() {
|
||||
assert_automatic_part_rollback(true).await;
|
||||
}
|
||||
|
||||
async fn assert_quorum_minus_one_retry_preserves_completable_part(
|
||||
disk_count: usize,
|
||||
parity: usize,
|
||||
success_indices: &[usize],
|
||||
) {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(disk_count, 0, parity).await;
|
||||
let bucket = format!("multipart-retry-{disk_count}-{}", success_indices[0]);
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, &bucket).await;
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(&bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
|
||||
let payload = vec![0x41; 1 << 20];
|
||||
let (upload_id, parts) =
|
||||
@@ -6489,12 +6870,18 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_upload_recovers_interrupted_part_retry() {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(6, 0, 2).await;
|
||||
let bucket = "multipart-retry-recovery";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
|
||||
let payload = vec![0x42; 1 << 20];
|
||||
let (upload_id, parts) =
|
||||
@@ -6654,10 +7041,15 @@ mod tests {
|
||||
bucket,
|
||||
object,
|
||||
&mut initial_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
&lifecycle_owned_put_options(
|
||||
&set_disks,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
.expect("initial object should be written");
|
||||
@@ -7068,7 +7460,7 @@ mod tests {
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let holder = set_disks
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await
|
||||
@@ -7104,7 +7496,7 @@ mod tests {
|
||||
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
|
||||
Err(StorageError::InvalidUploadID(..))
|
||||
));
|
||||
for temp_dir in temp_dirs {
|
||||
for temp_dir in &temp_dirs {
|
||||
assert!(
|
||||
!temp_dir
|
||||
.path()
|
||||
@@ -7133,7 +7525,7 @@ mod tests {
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartAfterRename);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -7170,9 +7562,11 @@ mod tests {
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn cancelled_put_object_part_keeps_upload_lock_through_commit() {
|
||||
for pause in [
|
||||
MultipartCommitPause::PutPartAfterCapacityAdmission,
|
||||
MultipartCommitPause::PutPartAfterRename,
|
||||
for (pause, typed_owner) in [
|
||||
(MultipartCommitPause::PutPartAfterCapacityAdmission, false),
|
||||
(MultipartCommitPause::PutPartAfterCapacityAdmission, true),
|
||||
(MultipartCommitPause::PutPartAfterRename, false),
|
||||
(MultipartCommitPause::PutPartAfterRename, true),
|
||||
] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
@@ -7188,7 +7582,18 @@ mod tests {
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let put_opts = if typed_owner {
|
||||
lifecycle_owned_put_options(&set_disks, bucket, ObjectOptions::default()).await
|
||||
} else {
|
||||
ObjectOptions::default()
|
||||
};
|
||||
let lifecycle_owner = put_opts.bucket_incarnation_fence.as_ref().map(Arc::downgrade);
|
||||
if typed_owner {
|
||||
assert!(put_opts.bucket_lifecycle_lock_fence.is_some());
|
||||
assert!(put_opts.bucket_lifecycle_lock_owner.is_none());
|
||||
assert!(lifecycle_owner.is_some());
|
||||
}
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, pause);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -7196,7 +7601,7 @@ mod tests {
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
|
||||
put_store
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &put_opts)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
@@ -7207,6 +7612,13 @@ mod tests {
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
if let Some(owner) = lifecycle_owner {
|
||||
assert!(
|
||||
owner.upgrade().is_some(),
|
||||
"cancelled caller must not release the commit's typed lifecycle owner"
|
||||
);
|
||||
}
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let mut abort = tokio::spawn(async move {
|
||||
@@ -7256,7 +7668,7 @@ mod tests {
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
// issue#5961: the barrier releases only once BOTH commits are paused
|
||||
// inside their commit sections, so reaching wait_until_paused proves the
|
||||
// two part numbers held their commit locks concurrently. Under an
|
||||
@@ -7352,7 +7764,7 @@ mod tests {
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_lock_path = format!("{upload_id_path}/part.1");
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartAfterRename);
|
||||
|
||||
let first_store = set_disks.clone();
|
||||
@@ -7461,7 +7873,7 @@ mod tests {
|
||||
let part_lock_path = format!("{upload_id_path}/part.1");
|
||||
*target.write().expect("lock-loss target should be writable") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, part_lock_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -7529,7 +7941,7 @@ mod tests {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
*target.write().expect("lock-loss target should be writable") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -7774,6 +8186,92 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_path_delete_flow_multipart_cleanup_preserves_order_and_quorum() {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations;
|
||||
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
for disk in &disks {
|
||||
for path in ["cleanup/first", "cleanup/blocker", "cleanup/last"] {
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, path, Bytes::from_static(b"staged"))
|
||||
.await
|
||||
.expect("seed raw multipart file");
|
||||
}
|
||||
}
|
||||
set.cleanup_multipart_path(&[
|
||||
"cleanup/first".to_owned(),
|
||||
"cleanup/blocker/child".to_owned(),
|
||||
"cleanup/last".to_owned(),
|
||||
])
|
||||
.await;
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, "cleanup/first").await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"batch keeps the successful item before its first error"
|
||||
);
|
||||
for path in ["cleanup/blocker", "cleanup/last"] {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, path)
|
||||
.await
|
||||
.expect("later file remains"),
|
||||
b"staged".as_slice(),
|
||||
"batch must stop at the first non-directory error"
|
||||
);
|
||||
}
|
||||
}
|
||||
set.cleanup_multipart_path(&["cleanup/blocker".to_owned(), "cleanup/last".to_owned()])
|
||||
.await;
|
||||
for disk in &disks {
|
||||
for path in ["cleanup/blocker", "cleanup/last"] {
|
||||
assert!(
|
||||
matches!(disk.read_all(RUSTFS_META_MULTIPART_BUCKET, path).await, Err(DiskError::FileNotFound)),
|
||||
"retry cleans remaining raw files on every disk"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let bucket = "path-delete-multipart";
|
||||
crate::bucket::metadata_sys::object_store_in(&set.ctx)
|
||||
.await
|
||||
.expect("fixture object store")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("authoritative multipart bucket");
|
||||
for available in [3, 2] {
|
||||
let object = format!("object-{available}");
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload before reducing availability");
|
||||
let saved = {
|
||||
let mut current = set.disks.write().await;
|
||||
let saved = current.clone();
|
||||
for disk in current.iter_mut().skip(available) {
|
||||
*disk = None;
|
||||
}
|
||||
saved
|
||||
};
|
||||
let result = set
|
||||
.abort_multipart_upload(bucket, &object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await;
|
||||
*set.disks.write().await = saved;
|
||||
let remaining = set.check_upload_id_exists(bucket, &object, &upload.upload_id, false).await;
|
||||
if available == 3 {
|
||||
result.expect("three of four disks satisfy delete write quorum");
|
||||
assert!(matches!(remaining, Err(StorageError::InvalidUploadID(..))));
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(StorageError::ErasureWriteQuorum)),
|
||||
"two disks must not acknowledge an abort"
|
||||
);
|
||||
remaining.expect("failed abort leaves quorum-visible staging after disks return");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_metadata_io_does_not_block_disk_registry_updates() {
|
||||
use crate::set_disk::core::io_primitives::rename_fanout_barrier::{self, PHASE_READ_VERSION};
|
||||
@@ -7940,7 +8438,7 @@ mod tests {
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x42; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let holder = set_disks
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await
|
||||
@@ -8025,7 +8523,7 @@ mod tests {
|
||||
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
|
||||
signaling.set_target(upload_resource.clone());
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
@@ -8165,7 +8663,7 @@ mod tests {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
@@ -8270,7 +8768,7 @@ mod tests {
|
||||
use futures::FutureExt;
|
||||
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async {
|
||||
let (dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let (mut dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-physical-namespace";
|
||||
let object = if undo { "undo-tail" } else { "publication-tail" };
|
||||
let old_body = vec![0x41; 1024];
|
||||
@@ -8440,7 +8938,7 @@ mod tests {
|
||||
if !coordinator_drained || !physical_drained || owner_drained.is_err() {
|
||||
// A bounded cleanup failure cannot justify deleting roots that a
|
||||
// detached executor might still use. Keep them for diagnosis.
|
||||
let retained = dirs.into_iter().map(TempDir::keep).collect::<Vec<_>>();
|
||||
let retained = dirs.keep_dirs();
|
||||
eprintln!("MPU cleanup incomplete: coordinator={coordinator_drained}, physical={physical_drained}, retained={retained:?}");
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
@@ -8611,7 +9109,7 @@ mod tests {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let completion_barrier =
|
||||
MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
|
||||
|
||||
@@ -8722,7 +9220,7 @@ mod tests {
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x65; 4096], &ObjectOptions::default()).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let completion_barrier =
|
||||
MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeObjectPublication);
|
||||
|
||||
@@ -8778,7 +9276,7 @@ mod tests {
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x49; 4096], &ObjectOptions::default()).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -8845,7 +9343,7 @@ mod tests {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
*target.write().expect("lock-loss target should be writable") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeLockLost);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
|
||||
+1505
-1008
File diff suppressed because it is too large
Load Diff
@@ -16,7 +16,7 @@
|
||||
//! another object (`a/b` and `a/b/c`). Cleanup of the parent must only remove
|
||||
//! the parent's own protocol data and never touch the child's xl.meta or data.
|
||||
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use super::hermetic_set_disks_support::{HermeticSetFixture, hermetic_set_disks_isolated};
|
||||
use super::*;
|
||||
use crate::data_movement::{SourceCleanupBucketFence, SourceCleanupError, cleanup_source_entry_if_unchanged};
|
||||
use rustfs_filemeta::FileInfoVersions;
|
||||
@@ -100,7 +100,7 @@ fn child_tree(temp_dirs: &[TempDir], bucket: &str) -> Vec<Vec<String>> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn setup(payload_len: usize) -> (Vec<TempDir>, Arc<SetDisks>, &'static str, Vec<u8>, Vec<u8>) {
|
||||
async fn setup(payload_len: usize) -> (HermeticSetFixture, Arc<SetDisks>, &'static str, Vec<u8>, Vec<u8>) {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "data-movement-prefix-cleanup";
|
||||
for disk in &disk_stores {
|
||||
|
||||
@@ -3885,8 +3885,7 @@ mod metadata_cache_tests {
|
||||
|
||||
let bucket = "late-metadata-identity";
|
||||
let object = "late-metadata-identity-object";
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await;
|
||||
let (_fixture, _disks, set) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create real late-read bucket");
|
||||
|
||||
@@ -8,6 +8,7 @@ pub(crate) mod admin {
|
||||
}
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use rustfs_storage_api::bucket::valid_artifact_base_prefix;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
BUCKET_LIFECYCLE_LOCK_OBJECT, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
SRBucketDeleteOp,
|
||||
|
||||
+274
-233
@@ -848,255 +848,293 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||
if !is_meta_bucketname(bucket)
|
||||
&& let Err(err) = check_valid_bucket_name_strict(bucket)
|
||||
{
|
||||
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
||||
}
|
||||
self.handle_make_bucket_inner(bucket, opts).await
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace.
|
||||
let bucket_lifecycle_guard = if !opts.no_lock {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
|
||||
Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
|
||||
Some(
|
||||
await_bucket_namespace_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket namespace lock acquisition",
|
||||
async {
|
||||
ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
|
||||
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: bucket.to_string(),
|
||||
object: bucket.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
}
|
||||
}
|
||||
other => StorageError::Lock(other),
|
||||
})
|
||||
},
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) => {
|
||||
if is_err_bucket_not_found(&err) {
|
||||
None
|
||||
} else {
|
||||
return Err(to_object_err(err, vec![bucket]));
|
||||
}
|
||||
// Keep the large creation future out of the instrumented handler's poll
|
||||
// frame. Guards remain owned by the same task.
|
||||
fn handle_make_bucket_inner<'a>(
|
||||
&'a self,
|
||||
bucket: &'a str,
|
||||
opts: &'a MakeBucketOptions,
|
||||
) -> futures::future::BoxFuture<'a, Result<()>> {
|
||||
Box::pin(async move {
|
||||
if opts.artifact_base_prefix.as_deref().is_some_and(|prefix| {
|
||||
is_meta_bucketname(bucket)
|
||||
|| opts.no_lock
|
||||
|| !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(prefix)
|
||||
}) {
|
||||
return Err(Error::other("invalid artifact bucket creation options"));
|
||||
}
|
||||
};
|
||||
let confirmed_missing = existing_bucket_info.is_none();
|
||||
let mut pending_creation_metadata = None;
|
||||
if confirmed_missing
|
||||
&& !opts.no_lock
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& let Some(metadata_store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await
|
||||
&& let Some(old_incarnation) = crate::bucket::metadata::load_bucket_incarnation(metadata_store, bucket).await?
|
||||
{
|
||||
// A quorum-level absence is not proof that every disk lost the old
|
||||
// generation. Require the explicit recovery path to verify all disks
|
||||
// and retire this incarnation before a same-name create can mutate
|
||||
// any volume.
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
StorageError::BucketExists(_) => StorageError::InvalidArgument(
|
||||
if !is_meta_bucketname(bucket)
|
||||
&& let Err(err) = check_valid_bucket_name_strict(bucket)
|
||||
{
|
||||
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace.
|
||||
let bucket_lifecycle_guard = if !opts.no_lock {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
|
||||
Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
|
||||
Some(
|
||||
await_bucket_namespace_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket namespace lock acquisition",
|
||||
async {
|
||||
ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
|
||||
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: bucket.to_string(),
|
||||
object: bucket.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
}
|
||||
}
|
||||
other => StorageError::Lock(other),
|
||||
})
|
||||
},
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) => {
|
||||
if is_err_bucket_not_found(&err) {
|
||||
None
|
||||
} else {
|
||||
return Err(to_object_err(err, vec![bucket]));
|
||||
}
|
||||
}
|
||||
};
|
||||
let confirmed_missing = existing_bucket_info.is_none();
|
||||
let mut pending_creation_metadata = None;
|
||||
if confirmed_missing
|
||||
&& !opts.no_lock
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& let Some(metadata_store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await
|
||||
&& let Some(old_incarnation) = crate::bucket::metadata::load_bucket_incarnation(metadata_store, bucket).await?
|
||||
{
|
||||
// A quorum-level absence is not proof that every disk lost the old
|
||||
// generation. Require the explicit recovery path to verify all disks
|
||||
// and retire this incarnation before a same-name create can mutate
|
||||
// any volume.
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
StorageError::BucketExists(_) => StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
"an old bucket volume remains on disk; administrator recovery refused".to_string(),
|
||||
),
|
||||
other => other,
|
||||
})?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let pending_intent = persisted
|
||||
&& metadata.bucket_incarnation_id == old_incarnation
|
||||
&& metadata.needs_bucket_creation_commit()
|
||||
&& !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
|
||||
if pending_intent {
|
||||
pending_creation_metadata = Some(metadata);
|
||||
} else {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
"an old bucket volume remains on disk; administrator recovery refused".to_string(),
|
||||
),
|
||||
other => other,
|
||||
})?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let pending_intent = persisted
|
||||
&& metadata.bucket_incarnation_id == old_incarnation
|
||||
&& metadata.needs_bucket_creation_commit()
|
||||
&& !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
|
||||
if pending_intent {
|
||||
pending_creation_metadata = Some(metadata);
|
||||
} else {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
|
||||
));
|
||||
}
|
||||
}
|
||||
let existing_metadata = if let Some(metadata) = pending_creation_metadata {
|
||||
Some(metadata)
|
||||
} else if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
metadata.created = existing_bucket_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.created)
|
||||
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// A sidecar left by an interrupted legacy migration is the
|
||||
// bucket's published incarnation; keep it rather than minting
|
||||
// one that would invalidate fences taken on the stored value.
|
||||
// A retired incarnation is residue of a deleted bucket and
|
||||
// must not come back.
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
|
||||
));
|
||||
}
|
||||
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
} else if !metadata.bucket_incarnation_sidecar {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
Some(metadata)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| {
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
BucketMetadata::new_with_default_durability(bucket)
|
||||
let existing_metadata = if let Some(metadata) = pending_creation_metadata {
|
||||
Some(metadata)
|
||||
} else if (opts.force_create || opts.artifact_base_prefix.is_some())
|
||||
&& !confirmed_missing
|
||||
&& !is_meta_bucketname(bucket)
|
||||
{
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
metadata.created = existing_bucket_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.created)
|
||||
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// A sidecar left by an interrupted legacy migration is the
|
||||
// bucket's published incarnation; keep it rather than minting
|
||||
// one that would invalidate fences taken on the stored value.
|
||||
// A retired incarnation is residue of a deleted bucket and
|
||||
// must not come back.
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
}
|
||||
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
} else if !metadata.bucket_incarnation_sidecar {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
Some(metadata)
|
||||
} else {
|
||||
BucketMetadata::new(bucket)
|
||||
}
|
||||
});
|
||||
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
|
||||
if confirmed_missing || is_meta_bucketname(bucket) {
|
||||
meta.set_created(opts.created_at);
|
||||
None
|
||||
};
|
||||
|
||||
if opts.versioning_enabled {
|
||||
let has_creation_identity = existing_metadata.is_some();
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| {
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
BucketMetadata::new_with_default_durability(bucket)
|
||||
} else {
|
||||
BucketMetadata::new(bucket)
|
||||
}
|
||||
});
|
||||
if let Some(prefix) = &opts.artifact_base_prefix {
|
||||
if has_creation_identity {
|
||||
if meta.artifact_base_prefix.as_ref() != Some(prefix) {
|
||||
return Err(Error::other("artifact namespace is immutable"));
|
||||
}
|
||||
} else if confirmed_missing {
|
||||
meta.artifact_base_prefix = Some(prefix.clone());
|
||||
} else {
|
||||
return Err(Error::other("artifact namespace requires a new bucket"));
|
||||
}
|
||||
}
|
||||
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
|
||||
if confirmed_missing || is_meta_bucketname(bucket) {
|
||||
meta.set_created(opts.created_at);
|
||||
|
||||
if opts.versioning_enabled || meta.artifact_base_prefix.is_some() {
|
||||
meta.versioning_config_xml =
|
||||
crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Object Lock enable is one-way, and it must apply to an existing bucket
|
||||
// too. Site replication replays make-with-versioning carrying the
|
||||
// source's lockEnabled against a destination bucket that already exists;
|
||||
// gating this on `confirmed_missing` returned success while leaving the
|
||||
// replica unlocked, so replicated versions could be deleted without the
|
||||
// retention the source enforces.
|
||||
let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
|
||||
if opts.lock_enabled {
|
||||
meta.lock_enabled = true;
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml =
|
||||
crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Object Lock enable is one-way, and it must apply to an existing bucket
|
||||
// too. Site replication replays make-with-versioning carrying the
|
||||
// source's lockEnabled against a destination bucket that already exists;
|
||||
// gating this on `confirmed_missing` returned success while leaving the
|
||||
// replica unlocked, so replicated versions could be deleted without the
|
||||
// retention the source enforces.
|
||||
let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
|
||||
if opts.lock_enabled {
|
||||
meta.lock_enabled = true;
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
|
||||
let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled;
|
||||
if metadata_persisted_before_physical {
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
|
||||
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
|
||||
}
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// Fence every scanner cycle that could have observed the namespace
|
||||
// before physical creation. Creation may become visible even when a
|
||||
// later metadata write or namespace-lock check fails.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
|
||||
}
|
||||
|
||||
if let Err(err) = await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"physical bucket creation",
|
||||
await_bucket_namespace_operation(
|
||||
metadata_transaction_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket creation metadata transaction",
|
||||
async {
|
||||
self.make_bucket_on_sets(bucket, opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if is_err_bucket_exists(&err)
|
||||
&& let Err(heal_err) = self
|
||||
.handle_heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
|
||||
}
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
|
||||
error!("make bucket failed: {err}");
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
};
|
||||
|
||||
let metadata_result = async {
|
||||
let metadata_persisted_before_physical =
|
||||
confirmed_missing && !is_meta_bucketname(bucket) && (opts.lock_enabled || meta.artifact_base_prefix.is_some());
|
||||
if metadata_persisted_before_physical {
|
||||
return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
|
||||
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
|
||||
}
|
||||
}
|
||||
if is_meta_bucketname(bucket) {
|
||||
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else if existing_incarnation_is_authoritative && !lock_newly_enabled {
|
||||
metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else {
|
||||
metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
if let Err(err) = metadata_result {
|
||||
if !metadata_lock_lost {
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
if metadata_lock_lost {
|
||||
return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// A scanner may have sampled the first fence before the bucket
|
||||
// became visible. Fence again after metadata initialization so
|
||||
// that snapshot cannot publish as complete.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// Fence every scanner cycle that could have observed the namespace
|
||||
// before physical creation. Creation may become visible even when a
|
||||
// later metadata write or namespace-lock check fails.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
// Keep cleanup state out of the bucket-creation request future.
|
||||
Box::pin(self.cleanup_bucket_usage(bucket, ns_guard.as_ref())).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
if let Err(err) = await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"physical bucket creation",
|
||||
await_bucket_namespace_operation(
|
||||
metadata_transaction_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket creation metadata transaction",
|
||||
async {
|
||||
self.make_bucket_on_sets(bucket, opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if is_err_bucket_exists(&err)
|
||||
&& let Err(heal_err) = self
|
||||
.handle_heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
|
||||
}
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost())
|
||||
{
|
||||
error!("make bucket failed: {err}");
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
};
|
||||
|
||||
let metadata_result = async {
|
||||
if metadata_persisted_before_physical {
|
||||
return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
|
||||
}
|
||||
if is_meta_bucketname(bucket) {
|
||||
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else if existing_incarnation_is_authoritative && !lock_newly_enabled {
|
||||
metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else {
|
||||
metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
if let Err(err) = metadata_result {
|
||||
if !metadata_lock_lost {
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
if metadata_lock_lost {
|
||||
return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// A scanner may have sampled the first fence before the bucket
|
||||
// became visible. Fence again after metadata initialization so
|
||||
// that snapshot cannot publish as complete.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Prove a live bucket generation before repairing missing expansion volumes.
|
||||
@@ -1461,6 +1499,9 @@ impl ECStore {
|
||||
}
|
||||
};
|
||||
if bucket_exists {
|
||||
if !is_meta_bucketname(bucket) {
|
||||
self.require_no_artifact_backup_retention(bucket).await?;
|
||||
}
|
||||
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
|
||||
|
||||
if !opts.force {
|
||||
|
||||
@@ -160,6 +160,7 @@ pub struct BucketIncarnationFenceGuard {
|
||||
bucket: String,
|
||||
token: Option<u64>,
|
||||
memoized: Option<Uuid>,
|
||||
validated_incarnation: Option<Uuid>,
|
||||
}
|
||||
|
||||
impl BucketIncarnationFenceGuard {
|
||||
@@ -173,6 +174,7 @@ impl BucketIncarnationFenceGuard {
|
||||
bucket: bucket.to_owned(),
|
||||
token,
|
||||
memoized,
|
||||
validated_incarnation: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,6 +185,7 @@ impl BucketIncarnationFenceGuard {
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<Uuid>>,
|
||||
{
|
||||
self.validated_incarnation = None;
|
||||
let cacheable = reuse_memo && self.token.is_some();
|
||||
let current = match self.memoized.filter(|_| cacheable) {
|
||||
Some(current) => current,
|
||||
@@ -197,6 +200,7 @@ impl BucketIncarnationFenceGuard {
|
||||
}
|
||||
};
|
||||
self.ensure_held()?;
|
||||
self.validated_incarnation = Some(current);
|
||||
Ok(current)
|
||||
}
|
||||
|
||||
@@ -213,6 +217,19 @@ impl BucketIncarnationFenceGuard {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, store: &super::ECStore, bucket: &str, incarnation: Uuid) -> bool {
|
||||
Arc::ptr_eq(&self.registry, &store.bucket_fence_registry)
|
||||
&& self.bucket == bucket
|
||||
&& self.validated_incarnation == Some(incarnation)
|
||||
&& !self.is_lock_lost()
|
||||
}
|
||||
|
||||
/// Retain this validated lifecycle owner through the storage write tail.
|
||||
pub fn attach_owner(self: &Arc<Self>, opts: &mut crate::object_api::ObjectOptions) {
|
||||
self.attach_to_object_options(opts);
|
||||
opts.bucket_incarnation_fence = Some(Arc::clone(self));
|
||||
}
|
||||
|
||||
/// Propagate lifecycle lock loss into the storage commit checks.
|
||||
/// The caller still owns this guard until the complete write tail drains.
|
||||
pub fn attach_to_object_options(&self, opts: &mut crate::object_api::ObjectOptions) {
|
||||
@@ -221,7 +238,8 @@ impl BucketIncarnationFenceGuard {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
/// Report whether the lifecycle lock no longer covers this operation.
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
}
|
||||
|
||||
@@ -401,6 +419,30 @@ mod tests {
|
||||
BucketIncarnationFenceGuard::new(registry, "b", inner, cacheable)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_owner_tracks_validation_instead_of_cached_incarnation() {
|
||||
let registry = Arc::new(BucketFenceRegistry::default());
|
||||
let lock = NamespaceLock::new("artifact-owner-validation".to_owned(), Arc::new(LocalClient::new()));
|
||||
let mut first = test_fence(registry.clone(), &lock, true).await;
|
||||
first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo");
|
||||
let mut snapshot = test_fence(registry, &lock, true).await;
|
||||
assert_eq!(snapshot.memoized, Some(uuid(7)));
|
||||
assert_eq!(snapshot.validated_incarnation, None, "registration alone grants no artifact ownership");
|
||||
snapshot
|
||||
.validate(false, || async { Ok(uuid(8)) })
|
||||
.await
|
||||
.expect("validate snapshot");
|
||||
assert_eq!(snapshot.validated_incarnation, Some(uuid(8)));
|
||||
assert_eq!(snapshot.memoized, Some(uuid(7)), "independent snapshots do not update the shared memo");
|
||||
assert!(
|
||||
snapshot
|
||||
.validate(false, || async { Err(StorageError::ErasureReadQuorum) })
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(snapshot.validated_incarnation, None, "failed validation cannot retain earlier ownership");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overlapping_validations_reuse_one_success_until_coverage_gap() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
@@ -39,6 +39,10 @@ pub(crate) struct BucketHealScope {
|
||||
}
|
||||
|
||||
impl BucketHealScope {
|
||||
pub(crate) fn lifecycle_guard(&self) -> Option<&NamespaceLockGuard> {
|
||||
self.fence.namespace_lock_guard()
|
||||
}
|
||||
|
||||
pub(crate) fn check(&self) -> disk::error::Result<()> {
|
||||
if self.fence.is_lock_lost() {
|
||||
return Err(DiskError::other("bucket heal incarnation fence was lost"));
|
||||
@@ -1070,16 +1074,25 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.handle_heal_object_with_absence(
|
||||
crate::bucket::artifact::with_artifact_repair_scope(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut None,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
Box::pin(async {
|
||||
self.handle_heal_object_with_absence(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut None,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1113,6 +1126,24 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_unversioned_absence: bool,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
crate::bucket::artifact::with_artifact_repair_scope(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
Box::pin(self.heal_object_with_absence_proof_inner(bucket, object, version_id, opts, allow_unversioned_absence)),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_absence_proof_inner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_unversioned_absence: bool,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
// Object heal may recreate a missing bucket volume before committing a
|
||||
// shard. Keep the current bucket generation fenced for the whole
|
||||
@@ -1131,7 +1162,11 @@ impl ECStore {
|
||||
// Reuse the admission's owner: reacquiring a read lock behind a queued
|
||||
// lifecycle writer would deadlock with the read lock we already hold.
|
||||
let scope = bucket_heal_scope(bucket).filter(|scope| std::ptr::eq(scope.store.as_ref(), self));
|
||||
let guard = if let Some(scope) = &scope {
|
||||
let repair = crate::bucket::artifact::artifact_repair_scope(bucket, object);
|
||||
let guard = if let Some(repair) = &repair {
|
||||
repair.check()?;
|
||||
None
|
||||
} else if let Some(scope) = &scope {
|
||||
scope.check().map_err(Error::from)?;
|
||||
None
|
||||
} else {
|
||||
@@ -1139,11 +1174,13 @@ impl ECStore {
|
||||
};
|
||||
let fence_is_lost = || {
|
||||
scope.as_ref().is_some_and(|scope| scope.fence.is_lock_lost())
|
||||
|| repair.as_ref().is_some_and(|scope| scope.check().is_err())
|
||||
|| guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
};
|
||||
let lifecycle_guard = scope
|
||||
.as_ref()
|
||||
.and_then(|scope| scope.fence.namespace_lock_guard())
|
||||
.or_else(|| repair.as_ref().and_then(|scope| scope.lifecycle_guard()))
|
||||
.or(guard.as_ref())
|
||||
.ok_or_else(|| Error::other("bucket heal requires a held lifecycle guard"))?;
|
||||
let retirement = crate::bucket::retirement::MarkerRetirementContext {
|
||||
@@ -2045,6 +2082,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4148,6 +4186,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
};
|
||||
|
||||
let err = store
|
||||
|
||||
@@ -428,6 +428,7 @@ impl ECStore {
|
||||
|
||||
let mut deployment_id = None;
|
||||
let mut pool_meta_bootstrap_authorities = Vec::new();
|
||||
let mut pending_format_disks = Vec::new();
|
||||
|
||||
// let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?;
|
||||
|
||||
@@ -553,6 +554,7 @@ impl ECStore {
|
||||
// formatted it, `LegacyAdoption` when it verified the migration, and
|
||||
// `None` when it merely read a format another node created.
|
||||
pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
|
||||
pending_format_disks.extend(loaded_format.pending_disks);
|
||||
let fm = loaded_format.format;
|
||||
|
||||
// Format loading succeeded, enable health monitoring on all disks
|
||||
@@ -618,6 +620,7 @@ impl ECStore {
|
||||
// this point) and later reads share one cell.
|
||||
ctx: instance_ctx.clone(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
|
||||
// Only set it when this instance's deployment ID is not yet configured
|
||||
@@ -627,8 +630,21 @@ impl ECStore {
|
||||
|
||||
let wait_sec = 5;
|
||||
let mut exit_count = 0;
|
||||
let disk_sets = ec
|
||||
.pools
|
||||
.iter()
|
||||
.flat_map(|pool| pool.disk_set.iter().cloned())
|
||||
.collect::<Vec<_>>();
|
||||
loop {
|
||||
if let Err(err) = ec.init(ctx.clone()).await {
|
||||
let initialization = async {
|
||||
ec.init(ctx.clone()).await?;
|
||||
for pending in &pending_format_disks {
|
||||
pending.finalize(&instance_ctx, &disk_sets).await?;
|
||||
}
|
||||
Ok::<(), Error>(())
|
||||
}
|
||||
.await;
|
||||
if let Err(err) = initialization {
|
||||
error!("init err: {}", err);
|
||||
error!("retry after {} second", wait_sec);
|
||||
sleep(Duration::from_secs(wait_sec)).await;
|
||||
@@ -652,6 +668,11 @@ impl ECStore {
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
// Keep startup state out of callers and the instrumented wrapper.
|
||||
Box::pin(self.init_inner(rx)).await
|
||||
}
|
||||
|
||||
async fn init_inner(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let should_persist_pool_meta = self
|
||||
@@ -869,6 +890,23 @@ impl ECStore {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_bucket_tests;
|
||||
#[cfg(unix)]
|
||||
mod artifact_inventory_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_put_seal_tests;
|
||||
#[cfg(all(feature = "test-util", unix))]
|
||||
mod artifact_receiver_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_repair_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_retirement_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_write_entrypoint_tests;
|
||||
#[cfg(all(feature = "test-util", unix))]
|
||||
mod artifact_writes_seal_tests;
|
||||
|
||||
use super::{
|
||||
LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
|
||||
establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
|
||||
@@ -1781,7 +1819,7 @@ mod tests {
|
||||
let mut pools = Vec::new();
|
||||
for _ in 0..4 {
|
||||
let (pool_dirs, pool_disks, pool) = hermetic_set_disks_isolated(4).await;
|
||||
dirs.extend(pool_dirs);
|
||||
dirs.push(pool_dirs);
|
||||
disks.push(pool_disks);
|
||||
pools.push(pool);
|
||||
}
|
||||
@@ -7298,8 +7336,8 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("seed older target metadata on all disks");
|
||||
let source = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let mut source = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("read source marker metadata")
|
||||
@@ -7308,6 +7346,30 @@ mod tests {
|
||||
.into_iter()
|
||||
.find(|version| version.version_id.unwrap_or_default() == marker_id)
|
||||
.expect("find source delete marker");
|
||||
if marker_incarnation.is_none() {
|
||||
// Public writes bind new markers to the current bucket generation.
|
||||
// Seed legacy disk metadata explicitly without weakening admission.
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut source.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_BUCKET_INCARNATION_ID,
|
||||
);
|
||||
for disk in source_set.disk_inventory().await.into_iter().flatten() {
|
||||
disk.write_metadata("", &bucket, object, source.clone())
|
||||
.await
|
||||
.expect("seed legacy source marker metadata");
|
||||
}
|
||||
source_set.invalidate_get_object_metadata_cache(&bucket, object).await;
|
||||
}
|
||||
let source = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("reload source marker metadata")
|
||||
.expect("source marker must remain present")
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.version_id.unwrap_or_default() == marker_id)
|
||||
.expect("find reloaded source delete marker");
|
||||
assert_eq!(source.delete_marker_incarnation(), marker_incarnation, "source fixture identity");
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
for disk in target_set.disk_inventory().await.into_iter().flatten().take(2) {
|
||||
disk.write_metadata("", &bucket, object, source.clone())
|
||||
@@ -7733,6 +7795,103 @@ mod tests {
|
||||
.expect_err("suspended delete must remove the requested UUID version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_retains_lifecycle_owner_through_source_cleanup() {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create decommission lifecycle-owner store");
|
||||
let (_ctx, store, shutdown) = Box::pin(without_storage_class_env(build_isolated_test_store(
|
||||
temp_dir.path(),
|
||||
"decommission-lifecycle-owner",
|
||||
&[4, 4],
|
||||
)))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = format!("decom-lifecycle-owner-{}", uuid::Uuid::new_v4());
|
||||
let object = "source.bin";
|
||||
let payload = vec![b'o'; 1_300_000];
|
||||
Box::pin(store.make_bucket(&bucket, &MakeBucketOptions::default()))
|
||||
.await
|
||||
.expect("create ordinary source bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("source bucket incarnation");
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
Box::pin(seed_decommission_source(&store, &bucket, object, payload.clone(), &opts)).await;
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let expected = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("load the exact cleanup snapshot")
|
||||
.expect("source object must exist");
|
||||
let source_disks = source_set.disk_inventory().await;
|
||||
assert_eq!(source_disks.iter().flatten().count(), 4, "all source disks must be online");
|
||||
let owner = store
|
||||
.acquire_bucket_incarnation_fence(&bucket, incarnation)
|
||||
.await
|
||||
.expect("hold source bucket lifecycle");
|
||||
let rejected = Box::pin(crate::data_movement::cleanup_source_entry_if_unchanged(
|
||||
Arc::clone(&source_set),
|
||||
&bucket,
|
||||
object,
|
||||
&expected,
|
||||
&[],
|
||||
&[],
|
||||
crate::data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: Some(incarnation),
|
||||
lifecycle_guard: owner.namespace_lock_guard(),
|
||||
..Default::default()
|
||||
},
|
||||
"test_unowned_source_cleanup",
|
||||
))
|
||||
.await
|
||||
.expect_err("a bare lifecycle signal must not authorize source cleanup");
|
||||
assert!(
|
||||
rejected
|
||||
.to_string()
|
||||
.contains("write requires an owned bucket lifecycle fence"),
|
||||
"cleanup must reject the missing owner: {rejected}"
|
||||
);
|
||||
let mut source = source_set
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("rejected cleanup must preserve the readable source");
|
||||
let mut remaining = Vec::new();
|
||||
source
|
||||
.stream
|
||||
.read_to_end(&mut remaining)
|
||||
.await
|
||||
.expect("drain preserved source");
|
||||
assert_eq!(remaining, payload);
|
||||
drop(source);
|
||||
drop(owner);
|
||||
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
Box::pin(run_decommission_entry_retry_test(
|
||||
&store,
|
||||
CancellationToken::new(),
|
||||
&bucket,
|
||||
object,
|
||||
Some(incarnation),
|
||||
Arc::new(AtomicUsize::new(0)),
|
||||
))
|
||||
.await
|
||||
.expect("decommission must copy and clean the ordinary bucket entry with its real owner");
|
||||
assert_eq!(read_decommission_target_body(&store, &bucket, object, &opts).await, payload);
|
||||
assert_decommission_source_absent(&store, &bucket, object, &opts).await;
|
||||
for disk in source_disks.into_iter().flatten() {
|
||||
assert!(
|
||||
!tokio::fs::try_exists(disk.path().join(&bucket).join(object))
|
||||
.await
|
||||
.expect("inspect the physical source path"),
|
||||
"source metadata and shards must be removed from every disk"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() {
|
||||
@@ -21073,10 +21232,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
|
||||
Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
|
||||
fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
|
||||
run_large_stack_async_test(
|
||||
"legacy-recovery-disposition",
|
||||
legacy_recovery_disposition_removes_only_local_journals_and_replays_case,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -0,0 +1,757 @@
|
||||
use super::{build_isolated_test_store, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{
|
||||
metadata::{
|
||||
BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, BucketMetadata,
|
||||
load_bucket_metadata_parse_with_presence,
|
||||
},
|
||||
metadata_sys,
|
||||
versioning::VersioningApi,
|
||||
},
|
||||
disk::{BUCKET_META_PREFIX, DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
runtime::instance::InstanceContext,
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
namespace::NamespaceLocking as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PREFIX: &str = "outputs/";
|
||||
const ENABLED: &[u8] = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";
|
||||
const SUSPENDED: &[u8] = b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>";
|
||||
const EXCLUDED_PREFIX: &[u8] = b"<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>outputs/</Prefix></ExcludedPrefixes></VersioningConfiguration>";
|
||||
const EXCLUDED_FOLDERS: &[u8] =
|
||||
b"<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>";
|
||||
const TAGGING: &[u8] = b"<Tagging><TagSet><Tag><Key>purpose</Key><Value>artifact-test</Value></Tag></TagSet></Tagging>";
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-bucket", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn drain_before_reopen(store: &ECStore, shutdown: &CancellationToken, bucket: &str) {
|
||||
shutdown.cancel();
|
||||
assert!(
|
||||
store.ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
|
||||
"ARTIFACT_PREVIOUS_STORE_MUTATIONS_DRAINED"
|
||||
);
|
||||
// Internal metadata PUTs can acknowledge quorum before their disk tails finish.
|
||||
// Acquire each key separately to wait for its actual writer to release the lock.
|
||||
for file in [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE] {
|
||||
let key = format!("{BUCKET_META_PREFIX}/{bucket}/{file}");
|
||||
let namespace = store.pools[0].disk_set[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &key)
|
||||
.await
|
||||
.expect("resolve persisted identity namespace");
|
||||
drop(
|
||||
namespace
|
||||
.get_write_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("identity write tail drained"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn dedicated_bucket(store: &ECStore, bucket: &str) {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some(PREFIX.to_owned()),
|
||||
// Creation must enforce versioning without relying on the request flag.
|
||||
versioning_enabled: false,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a dedicated artifact bucket on real disks");
|
||||
}
|
||||
|
||||
async fn persisted(store: &Arc<ECStore>, bucket: &str) -> BucketMetadata {
|
||||
let (metadata, exists) = load_bucket_metadata_parse_with_presence(store.clone(), bucket, true)
|
||||
.await
|
||||
.expect("read actual persisted bucket metadata");
|
||||
assert!(exists, "ARTIFACT_METADATA_PERSISTED");
|
||||
metadata
|
||||
}
|
||||
|
||||
fn assert_namespace(metadata: &BucketMetadata, incarnation: Uuid) {
|
||||
assert_eq!(metadata.artifact_base_prefix.as_deref(), Some(PREFIX), "ARTIFACT_NAMESPACE_IMMUTABLE");
|
||||
assert_eq!(metadata.bucket_incarnation_id, incarnation, "ARTIFACT_INCARNATION_IMMUTABLE");
|
||||
assert!(
|
||||
metadata.versioning_config.as_ref().is_some_and(VersioningApi::enabled),
|
||||
"ARTIFACT_VERSIONING_REQUIRED"
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_persisted_and_cached(store: &Arc<ECStore>, bucket: &str, incarnation: Uuid) {
|
||||
assert_namespace(&persisted(store, bucket).await, incarnation);
|
||||
let cached = metadata_sys::get_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read cached bucket identity");
|
||||
assert_namespace(&cached, incarnation);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_creation_enables_versioning_and_survives_store_reopen() {
|
||||
let root = tempfile::tempdir().expect("artifact bucket store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-created-reopen";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let created = persisted(&store, bucket).await;
|
||||
let incarnation = created.bucket_incarnation_id;
|
||||
assert!(!incarnation.is_nil(), "ARTIFACT_INCARNATION_REQUIRED");
|
||||
assert!(created.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
assert_namespace(&created, incarnation);
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
metadata_sys::reload_bucket_metadata(store.clone(), bucket)
|
||||
.await
|
||||
.expect("load the reopened bucket from persisted metadata");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary force-create replay preserves the dedicated namespace");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_ordinary_configuration_update_preserves_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact configuration store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-config-update";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let incarnation = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
metadata_sys::update_in(&store.ctx, bucket, BUCKET_TAGGING_CONFIG, TAGGING.to_vec())
|
||||
.await
|
||||
.expect("ordinary tagging update remains supported");
|
||||
let changed = persisted(&store, bucket).await;
|
||||
assert_eq!(changed.tagging_config_xml, TAGGING, "ARTIFACT_ORDINARY_CONFIG_UPDATED");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let suspended = crate::bucket::utils::deserialize(SUSPENDED).expect("valid suspended versioning XML");
|
||||
for stale_typed_config in [None, Some(suspended)] {
|
||||
let mut replacement = changed.clone();
|
||||
replacement.versioning_config = stale_typed_config;
|
||||
sys.set_validated(replacement)
|
||||
.await
|
||||
.expect("validated cache setter reparses authoritative XML");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_create_rejects_existing_bucket_type_and_prefix_changes() {
|
||||
let root = tempfile::tempdir().expect("artifact immutable creation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let ordinary = "artifact-existing-ordinary";
|
||||
store
|
||||
.make_bucket(ordinary, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create ordinary bucket");
|
||||
let original = persisted(&store, ordinary).await;
|
||||
assert!(original.artifact_base_prefix.is_none());
|
||||
for force_create in [false, true] {
|
||||
let result = store
|
||||
.make_bucket(
|
||||
ordinary,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some(PREFIX.to_owned()),
|
||||
force_create,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: ordinary bucket cannot change type");
|
||||
let unchanged = persisted(&store, ordinary).await;
|
||||
assert_eq!(unchanged.artifact_base_prefix, None);
|
||||
assert_eq!(unchanged.bucket_incarnation_id, original.bucket_incarnation_id);
|
||||
assert_eq!(unchanged.versioning_config_xml, original.versioning_config_xml);
|
||||
}
|
||||
let dedicated = "artifact-existing-dedicated";
|
||||
dedicated_bucket(&store, dedicated).await;
|
||||
let incarnation = persisted(&store, dedicated).await.bucket_incarnation_id;
|
||||
for force_create in [false, true] {
|
||||
let result = store
|
||||
.make_bucket(
|
||||
dedicated,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("another/".to_owned()),
|
||||
force_create,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: create cannot replace an existing prefix");
|
||||
assert_persisted_and_cached(&store, dedicated, incarnation).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_full_metadata_replacement_cannot_change_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact metadata replacement store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let dedicated = "artifact-replacement-dedicated";
|
||||
let ordinary = "artifact-replacement-ordinary";
|
||||
dedicated_bucket(&store, dedicated).await;
|
||||
store
|
||||
.make_bucket(ordinary, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create ordinary replacement fixture");
|
||||
let original = persisted(&store, dedicated).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for route in 0..3 {
|
||||
for prefix in [None, Some("replacement/".to_owned())] {
|
||||
let mut replacement = original.clone();
|
||||
replacement.artifact_base_prefix = prefix;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_NAMESPACE_IMMUTABLE: complete replacement cannot remove or change prefix"
|
||||
);
|
||||
assert_persisted_and_cached(&store, dedicated, incarnation).await;
|
||||
}
|
||||
let mut replacement = persisted(&store, ordinary).await;
|
||||
let ordinary_incarnation = replacement.bucket_incarnation_id;
|
||||
replacement.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
replacement.versioning_config_xml = ENABLED.to_vec();
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_NAMESPACE_IMMUTABLE: complete replacement cannot promote ordinary bucket"
|
||||
);
|
||||
let unchanged = persisted(&store, ordinary).await;
|
||||
assert!(unchanged.artifact_base_prefix.is_none());
|
||||
assert_eq!(unchanged.bucket_incarnation_id, ordinary_incarnation);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_versioning_cannot_be_suspended_or_removed() {
|
||||
let root = tempfile::tempdir().expect("artifact mandatory versioning store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-required-versioning";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let original = persisted(&store, bucket).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for xml in [SUSPENDED, b"".as_slice(), EXCLUDED_PREFIX, EXCLUDED_FOLDERS] {
|
||||
let result = metadata_sys::update_in(&store.ctx, bucket, BUCKET_VERSIONING_CONFIG, xml.to_vec()).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED: configuration update cannot disable versioning"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
for route in 0..3 {
|
||||
let mut replacement = original.clone();
|
||||
replacement.versioning_config_xml = xml.to_vec();
|
||||
replacement.versioning_config = None;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED: complete replacement cannot disable versioning"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_pending_creation_reopen_retry_preserves_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact pending creation store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-pending-retry";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(bucket);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
let incarnation = intent.bucket_incarnation_id;
|
||||
// Persist through the production intent writer, before any physical bucket exists.
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("persist real creation intent");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert!(!pending.bucket_creation_committed, "ARTIFACT_CREATION_PENDING");
|
||||
assert_namespace(&pending, incarnation);
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists(),
|
||||
"ARTIFACT_PHYSICAL_BUCKET_NOT_CREATED"
|
||||
);
|
||||
}
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let result = store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("replacement/".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: pending intent rejects a changed request");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert_namespace(&pending, incarnation);
|
||||
assert!(!pending.bucket_creation_committed);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("retry resumes persisted creation identity");
|
||||
let completed = persisted(&store, bucket).await;
|
||||
assert!(completed.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).is_dir(),
|
||||
"ARTIFACT_PHYSICAL_BUCKET_CREATED"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_partial_volume_retry_preserves_pending_identity() {
|
||||
let root = tempfile::tempdir().expect("artifact partial volume store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-partial-volume";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(bucket);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
let incarnation = intent.bucket_incarnation_id;
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("persist real creation intent before a partial physical creation");
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("first real disk")
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("create only one physical volume");
|
||||
assert!(root.path().join(format!("pool0/set0/disk0/{bucket}")).is_dir());
|
||||
for disk in 1..4 {
|
||||
assert!(!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists());
|
||||
}
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect_err("partial physical creation requires explicit recovery");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert_namespace(&pending, incarnation);
|
||||
assert!(!pending.bucket_creation_committed, "ARTIFACT_PARTIAL_CREATION_STAYS_PENDING");
|
||||
for disk in 1..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists(),
|
||||
"ARTIFACT_PARTIAL_CREATION_NOT_SILENTLY_COMPLETED"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_metadata_replacement_cannot_spoof_incarnation() {
|
||||
let root = tempfile::tempdir().expect("artifact spoofed incarnation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-spoofed-incarnation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let original = persisted(&store, bucket).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for route in 0..4 {
|
||||
for spoofed in [Uuid::nil(), Uuid::new_v4()] {
|
||||
for (prefix, xml) in [
|
||||
(None, b"".as_slice()),
|
||||
(None, SUSPENDED),
|
||||
(Some("replacement/"), ENABLED),
|
||||
(Some(PREFIX), ENABLED),
|
||||
] {
|
||||
let mut replacement = original.clone();
|
||||
replacement.bucket_incarnation_id = spoofed;
|
||||
replacement.artifact_base_prefix = prefix.map(str::to_owned);
|
||||
replacement.versioning_config_xml = xml.to_vec();
|
||||
replacement.versioning_config = None;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
2 => sys.set_validated(replacement).await,
|
||||
_ => replacement.save_with_store(store.clone()).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_INCARNATION_IMMUTABLE: route={route}, spoofed={spoofed}, prefix={prefix:?}"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_real_delete_recreate_allows_new_namespace_and_incarnation() {
|
||||
let root = tempfile::tempdir().expect("artifact genuine recreation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-genuine-recreation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let mut incarnation = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
for prefix in [Some("replacement/"), None] {
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete the empty bucket through production storage");
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: prefix.map(str::to_owned),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("genuine recreation may choose a new bucket namespace");
|
||||
let recreated = persisted(&store, bucket).await;
|
||||
assert!(!recreated.bucket_incarnation_id.is_nil(), "ARTIFACT_INCARNATION_REQUIRED");
|
||||
assert_ne!(recreated.bucket_incarnation_id, incarnation, "ARTIFACT_RECREATION_NEW_INCARNATION");
|
||||
assert_eq!(recreated.artifact_base_prefix.as_deref(), prefix, "ARTIFACT_RECREATION_NEW_NAMESPACE");
|
||||
assert_eq!(recreated.versioning_config.as_ref().is_some_and(VersioningApi::enabled), prefix.is_some());
|
||||
assert!(recreated.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
let cached = metadata_sys::get_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("recreated identity cached");
|
||||
assert_eq!(cached.bucket_incarnation_id, recreated.bucket_incarnation_id);
|
||||
assert_eq!(cached.artifact_base_prefix.as_deref(), prefix);
|
||||
incarnation = recreated.bucket_incarnation_id;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn artifact_bucket_create_observes_current_generation() {
|
||||
let root = tempfile::tempdir().expect("creation observation storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-create-observation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let initial = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete before observation");
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let current = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
assert_ne!(initial, current);
|
||||
// Invoke the exact observation phase consumed by make_artifact_bucket,
|
||||
// after real replacement between its creation and observation phases.
|
||||
assert_eq!(
|
||||
store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("observe replacement"),
|
||||
current,
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION"
|
||||
);
|
||||
assert!(
|
||||
store.observe_artifact_bucket_incarnation(bucket, "other/").await.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: mismatched namespace cannot produce a receipt"
|
||||
);
|
||||
let attempt = format!("{PREFIX}61/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, initial, &attempt)
|
||||
.await
|
||||
.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: old incarnation cannot seal same prefix"
|
||||
);
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, current, &attempt)
|
||||
.await
|
||||
.expect("current identity seals");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, reopened, stop) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
reopened
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("reopen observed identity"),
|
||||
current,
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION"
|
||||
);
|
||||
stop.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn artifact_bucket_create_observation_rejects_corrupt_authority() {
|
||||
use crate::{
|
||||
config::com::{read_config, save_config_with_opts},
|
||||
object_api::{ObjectOptions, WriteCompletion},
|
||||
};
|
||||
let root = tempfile::tempdir().expect("creation observation faults");
|
||||
let (mut ctx, mut store, mut shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-create-corruption";
|
||||
let incarnation = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("actual create and observation");
|
||||
let original = persisted(&store, bucket).await;
|
||||
let sys = ctx.bucket_metadata_sys().expect("actual metadata authority");
|
||||
// These faults target the authoritative cache branch actually read by the
|
||||
// observation. Public metadata updates reject them before publication.
|
||||
for fault in 0..6 {
|
||||
let mut damaged = original.clone();
|
||||
match fault {
|
||||
0 => damaged.name = "wrong-bucket".into(),
|
||||
1 => damaged.artifact_base_prefix = Some("other/".into()),
|
||||
2 => damaged.versioning_config = None,
|
||||
3 => damaged.versioning_config.as_mut().expect("versioning").exclude_folders = Some(true),
|
||||
4 => damaged.versioning_config = Some(crate::bucket::utils::deserialize(SUSPENDED).expect("suspended XML")),
|
||||
_ => {
|
||||
damaged.versioning_config = Some(crate::bucket::utils::deserialize(EXCLUDED_PREFIX).expect("excluded prefix XML"))
|
||||
}
|
||||
}
|
||||
sys.read().await.set_for_test(bucket.into(), Arc::new(damaged)).await;
|
||||
assert!(
|
||||
store.observe_artifact_bucket_incarnation(bucket, PREFIX).await.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: corrupt cached authority fault={fault}"
|
||||
);
|
||||
sys.read().await.set_for_test(bucket.into(), Arc::new(original.clone())).await;
|
||||
}
|
||||
drop(sys);
|
||||
for (file, corrupt, expected_error) in [
|
||||
(BUCKET_METADATA_FILE, b"corrupt metadata".to_vec(), "read_bucket_metadata: format invalid"),
|
||||
(
|
||||
BUCKET_INCARNATION_FILE,
|
||||
Uuid::nil().as_bytes().to_vec(),
|
||||
"persisted bucket incarnation is nil",
|
||||
),
|
||||
] {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{bucket}/{file}");
|
||||
let saved = read_config(store.clone(), &path).await.expect("real persisted record");
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
save_config_with_opts(store.clone(), &path, corrupt, &opts)
|
||||
.await
|
||||
.expect("inject physical record fault");
|
||||
// Reopen with a fresh metadata authority so observation must read the
|
||||
// actual damaged record, without a cached value or a negative entry.
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
(ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let error = store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("physical corruption cannot produce a receipt");
|
||||
assert!(
|
||||
error.to_string().contains(expected_error),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: corrupt physical authority {file}: {error}"
|
||||
);
|
||||
save_config_with_opts(store.clone(), &path, saved, &opts)
|
||||
.await
|
||||
.expect("restore physical record");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
(ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("restored observation"),
|
||||
incarnation
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_create_requires_namespace_locking_before_creation() {
|
||||
temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
|
||||
let root = tempfile::tempdir().expect("disabled lock storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert!(ctx.lock_manager().is_disabled());
|
||||
let bucket = "artifact-create-no-lock";
|
||||
let error = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("creation requires real locks");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket creation requires namespace locking")
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"ARTIFACT_CREATE_LOCK_REQUIRED: rejection precedes physical creation"
|
||||
);
|
||||
shutdown.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_inspect_reopens_without_migrating_other_buckets() {
|
||||
use crate::config::com::read_config;
|
||||
let root = tempfile::tempdir().expect("inspect storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-inspect-reopen";
|
||||
let incarnation = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("managed bucket");
|
||||
let pending = "artifact-inspect-pending";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(pending);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("real pending intent");
|
||||
// Physical quorum exists: a migrating reader could silently complete this intent.
|
||||
let legacy = "artifact-inspect-legacy";
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await.into_iter().flatten() {
|
||||
for name in [legacy, pending] {
|
||||
disk.make_volume(name).await.expect("real preexisting volume");
|
||||
}
|
||||
}
|
||||
let pending_path = format!("{BUCKET_META_PREFIX}/{pending}/{BUCKET_METADATA_FILE}");
|
||||
let before = read_config(store.clone(), &pending_path).await.expect("pending bytes");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drain_before_reopen(&store, &shutdown, pending).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.inspect_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("fresh-instance inspection"),
|
||||
incarnation
|
||||
);
|
||||
for name in [legacy, pending, "artifact-inspect-absent"] {
|
||||
assert!(store.inspect_artifact_bucket(name, PREFIX).await.is_err(), "ARTIFACT_INSPECT_READ_ONLY");
|
||||
}
|
||||
assert_eq!(
|
||||
read_config(store.clone(), &pending_path).await.expect("pending remains"),
|
||||
before,
|
||||
"ARTIFACT_INSPECT_READ_ONLY"
|
||||
);
|
||||
assert!(!persisted(&store, pending).await.bucket_creation_committed);
|
||||
for name in [legacy, "artifact-inspect-absent"] {
|
||||
for file in [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE] {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{name}/{file}");
|
||||
assert!(
|
||||
read_config(store.clone(), &path).await.is_err(),
|
||||
"ARTIFACT_INSPECT_READ_ONLY: no migration record"
|
||||
);
|
||||
}
|
||||
}
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
!root
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk}/artifact-inspect-absent"))
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
assert!(store.inspect_artifact_bucket(bucket, "wrong/").await.is_err());
|
||||
assert_eq!(store.inspect_artifact_bucket(bucket, PREFIX).await.unwrap(), incarnation);
|
||||
store
|
||||
.ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
|
||||
.await;
|
||||
let error = store
|
||||
.inspect_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("file endpoints cannot use distributed lock mode");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact PUT sealing requires local storage or a distributed Unix URL topology")
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_inspect_requires_namespace_locking() {
|
||||
temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
|
||||
let root = tempfile::tempdir().expect("disabled inspect locking");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert!(ctx.lock_manager().is_disabled());
|
||||
let error = store
|
||||
.inspect_artifact_bucket("artifact-inspect-no-lock", PREFIX)
|
||||
.await
|
||||
.expect_err("locking required before reading");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket inspection requires namespace locking")
|
||||
);
|
||||
shutdown.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::metadata_sys,
|
||||
disk::{Disk, local::artifact_inventory::ArtifactInventoryFrame},
|
||||
object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_managed_put_zero_and_versions_are_complete() {
|
||||
assert_managed_put_inventory_is_complete(ShardIntegrityWriteMode::Legacy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_protected_managed_put_versions_are_complete() {
|
||||
assert_managed_put_inventory_is_complete(ShardIntegrityWriteMode::Protected);
|
||||
}
|
||||
|
||||
fn assert_managed_put_inventory_is_complete(mode: ShardIntegrityWriteMode) {
|
||||
run_large_stack_async_test("artifact-inventory-managed", move || async move {
|
||||
let root = tempfile::tempdir().expect("managed inventory disk roots");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "inventory-managed", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "artifact-inventory-managed";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("actual managed versioned bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("actual incarnation");
|
||||
let prefix = format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
let keys = vec![
|
||||
(format!("{prefix}zero"), Vec::new()),
|
||||
(format!("{prefix}inline"), vec![0xA5; 1024]),
|
||||
(format!("{prefix}payload"), vec![0xA5; 1024 * 1024]),
|
||||
];
|
||||
let mut expected = std::collections::BTreeSet::new();
|
||||
for (key, data) in &keys {
|
||||
let object = store
|
||||
.put_object(
|
||||
bucket,
|
||||
key,
|
||||
&mut PutObjReader::from_vec(data.clone()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("real managed PUT including empty, inline and sharded payloads");
|
||||
expected.insert((key.clone(), object.version_id.expect("exact managed version")));
|
||||
}
|
||||
let identity = store
|
||||
.artifact_receiver_identity(bucket, incarnation, &prefix, true)
|
||||
.await
|
||||
.expect("metadata-derived identity");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_writes(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("actual all-writes seal");
|
||||
for member in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let member = member.expect("configured local member");
|
||||
let Disk::Local(local) = member.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let local = local.get_disk();
|
||||
let id = local.artifact_disk_id().await.expect("fresh actual disk identity");
|
||||
let (output, mut input) = tokio::sync::mpsc::channel(2);
|
||||
let (result, frames) = tokio::join!(
|
||||
local.inventory_artifact_attempt(
|
||||
id,
|
||||
bucket,
|
||||
&prefix,
|
||||
identity,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output
|
||||
),
|
||||
async {
|
||||
let mut frames = Vec::new();
|
||||
while let Some(frame) = input.recv().await {
|
||||
frames.push(frame);
|
||||
}
|
||||
frames
|
||||
}
|
||||
);
|
||||
assert!(result.is_ok(), "A03I1_REAL_MANAGED_PUT_SCAN: {result:?}");
|
||||
let actual = frames
|
||||
.iter()
|
||||
.filter_map(|frame| match frame {
|
||||
ArtifactInventoryFrame::Entries { versions, .. } => Some(versions),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.map(|version| (version.key.clone(), version.version_id))
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(actual, expected, "A03I1_REAL_ZERO_INLINE_AND_SHARDED_VERSIONS");
|
||||
assert!(matches!(frames.last(), Some(ArtifactInventoryFrame::Complete { entries: 3, .. })));
|
||||
}
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
async fn inventory_fixture() -> (
|
||||
tempfile::TempDir,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
tokio_util::sync::CancellationToken,
|
||||
Uuid,
|
||||
String,
|
||||
) {
|
||||
let root = tempfile::tempdir().expect("inventory aggregate roots");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "inventory-aggregate", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(
|
||||
"inventory-aggregate",
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".into()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("managed bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id("inventory-aggregate")
|
||||
.await
|
||||
.expect("actual incarnation");
|
||||
let prefix = format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
store
|
||||
.put_object(
|
||||
"inventory-aggregate",
|
||||
&format!("{prefix}object"),
|
||||
&mut PutObjReader::from_vec(vec![1; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("actual managed object");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_writes("inventory-aggregate", incarnation, &prefix)
|
||||
.await
|
||||
.expect("actual two-domain seal");
|
||||
(root, store, shutdown, incarnation, prefix)
|
||||
}
|
||||
|
||||
async fn aggregate(
|
||||
store: std::sync::Arc<crate::store::ECStore>,
|
||||
incarnation: Uuid,
|
||||
prefix: &str,
|
||||
) -> (
|
||||
crate::error::Result<()>,
|
||||
Vec<crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame>,
|
||||
) {
|
||||
let (output, mut receiver) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async {
|
||||
let mut frames = Vec::new();
|
||||
while let Some(frame) = receiver.recv().await {
|
||||
frames.push(frame);
|
||||
}
|
||||
frames
|
||||
};
|
||||
tokio::join!(
|
||||
store.inventory_artifact_attempt(
|
||||
"inventory-aggregate",
|
||||
incarnation,
|
||||
prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output
|
||||
),
|
||||
drain
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_all_members_required() {
|
||||
run_large_stack_async_test("artifact-inventory-all-members", || async {
|
||||
use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let (result, frames) = aggregate(store.clone(), incarnation, &prefix).await;
|
||||
result.expect("all four real sealed disks scanned");
|
||||
let observed: std::collections::BTreeSet<_> = frames
|
||||
.iter()
|
||||
.filter_map(|frame| match frame {
|
||||
ArtifactAttemptInventoryFrame::Observations {
|
||||
endpoint,
|
||||
disk_id,
|
||||
versions,
|
||||
} => {
|
||||
assert_eq!(versions.len(), 1);
|
||||
assert_eq!(versions[0].key, format!("{prefix}object"));
|
||||
Some((endpoint.clone(), *disk_id))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(observed.len(), 4, "A03I2_ALL_MEMBERS");
|
||||
assert!(
|
||||
matches!(
|
||||
frames.last(),
|
||||
Some(ArtifactAttemptInventoryFrame::Complete {
|
||||
members: 4,
|
||||
observations: 4
|
||||
})
|
||||
),
|
||||
"A03I2_ALL_MEMBERS"
|
||||
);
|
||||
let original = store.pools[0].disk_set[0].disks.write().await[3].take();
|
||||
let (denied, frames) = aggregate(store.clone(), incarnation, &prefix).await;
|
||||
assert!(denied.is_err(), "one missing configured member is incomplete");
|
||||
assert!(
|
||||
!frames
|
||||
.iter()
|
||||
.any(|frame| matches!(frame, ArtifactAttemptInventoryFrame::Complete { .. }))
|
||||
);
|
||||
store.pools[0].disk_set[0].disks.write().await[3] = original;
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_membership_change_is_incomplete() {
|
||||
run_large_stack_async_test("artifact-inventory-membership", || async {
|
||||
use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let original = store.pools[0].disk_set[0].disk_inventory().await[0]
|
||||
.clone()
|
||||
.expect("first member");
|
||||
let Disk::Local(local) = original.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let replacement = std::sync::Arc::new(Disk::Local(local.clone()));
|
||||
let (output, mut receiver) = tokio::sync::mpsc::channel(1);
|
||||
let owned = store.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
owned
|
||||
.inventory_artifact_attempt(
|
||||
"inventory-aggregate",
|
||||
incarnation,
|
||||
&prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output,
|
||||
)
|
||||
.await
|
||||
});
|
||||
assert!(matches!(receiver.recv().await, Some(ArtifactAttemptInventoryFrame::Observations { .. })));
|
||||
store.pools[0].disk_set[0].disks.write().await[0] = Some(replacement);
|
||||
let mut complete = false;
|
||||
while let Some(frame) = receiver.recv().await {
|
||||
complete |= matches!(frame, ArtifactAttemptInventoryFrame::Complete { .. });
|
||||
}
|
||||
let error = worker
|
||||
.await
|
||||
.expect("aggregate worker")
|
||||
.expect_err("same endpoint/UUID but changed captured Arc is incomplete");
|
||||
assert!(error.to_string().contains("membership changed"), "{error}");
|
||||
assert!(!complete);
|
||||
store.pools[0].disk_set[0].disks.write().await[0] = Some(original);
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_readonly_authority_cold_cache_and_queued_writer() {
|
||||
run_large_stack_async_test("artifact-inventory-readonly-authority", || async {
|
||||
use crate::{
|
||||
disk::{BUCKET_META_PREFIX, DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
storage_api_contracts::namespace::NamespaceLocking as _,
|
||||
};
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let member = store.pools[0].disk_set[0].disk_inventory().await[0]
|
||||
.clone()
|
||||
.expect("real receiver");
|
||||
let Disk::Local(local) = member.as_ref() else {
|
||||
panic!("local");
|
||||
};
|
||||
let expected = local.artifact_disk_id().await.expect("actual disk");
|
||||
let _lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock("inventory-aggregate")
|
||||
.await
|
||||
.expect("coordinator lifecycle owner");
|
||||
metadata_sys::remove_bucket_metadata_in(&store.ctx, "inventory-aggregate")
|
||||
.await
|
||||
.expect("cold metadata cache");
|
||||
let lock = store
|
||||
.new_ns_lock("inventory-aggregate", "inventory-aggregate")
|
||||
.await
|
||||
.expect("business bucket namespace");
|
||||
let reader = lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("existing namespace reader");
|
||||
let mut writer = Box::pin(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()));
|
||||
assert!(
|
||||
matches!(futures::poll!(writer.as_mut()), std::task::Poll::Pending),
|
||||
"real queued business bucket WRITE"
|
||||
);
|
||||
let (output, mut frames) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async { while frames.recv().await.is_some() {} };
|
||||
let endpoint = member.endpoint().to_string();
|
||||
let receive = store.inventory_local_artifact_attempt(
|
||||
(&endpoint, expected),
|
||||
("inventory-aggregate", incarnation),
|
||||
&prefix,
|
||||
rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase::ArtifactInventoryScan,
|
||||
tokio::time::Instant::now() + Duration::from_secs(2),
|
||||
output,
|
||||
);
|
||||
let (result, ()) = tokio::join!(receive, drain);
|
||||
result.expect("A03I2_READONLY_AUTHORITY_NO_REENTRY");
|
||||
drop(writer);
|
||||
drop(reader);
|
||||
drop(_lifecycle);
|
||||
// A legacy physical bucket lacking both authoritative records must
|
||||
// stay legacy after rejected inventory; no implicit getter migration.
|
||||
store
|
||||
.make_bucket(
|
||||
"inventory-legacy",
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary legacy fixture");
|
||||
let members = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
for member in members.iter().flatten() {
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
let path = member
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join("inventory-legacy")
|
||||
.join(file);
|
||||
if path.exists() {
|
||||
std::fs::remove_dir_all(&path).expect("remove fixture-only creation facts");
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata_sys::remove_bucket_metadata_in(&store.ctx, "inventory-legacy")
|
||||
.await
|
||||
.expect("cold legacy cache");
|
||||
let (output, mut frames) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async { while frames.recv().await.is_some() {} };
|
||||
let (denied, ()) = tokio::join!(
|
||||
store.clone().inventory_artifact_attempt(
|
||||
"inventory-legacy",
|
||||
Uuid::new_v4(),
|
||||
&prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(2),
|
||||
output
|
||||
),
|
||||
drain
|
||||
);
|
||||
assert!(denied.is_err());
|
||||
for member in members.iter().flatten() {
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
assert!(
|
||||
!member
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join("inventory-legacy")
|
||||
.join(file)
|
||||
.exists(),
|
||||
"A03I2_READONLY_NO_MIGRATION"
|
||||
);
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,587 @@
|
||||
use super::{build_isolated_test_store, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{artifact::artifact_put_seal_record_path, metadata_sys},
|
||||
config::com,
|
||||
disk::{DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
object_api::{ObjectInfo, ObjectOptions, PutObjReader, WriteCompletion},
|
||||
runtime::instance::InstanceContext,
|
||||
set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, rename_fanout_barrier},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEADLINE: Duration = Duration::from_secs(30);
|
||||
|
||||
fn attempt_prefix() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-put-seal", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn create_bucket(store: &ECStore, bucket: &str) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a real versioned artifact bucket");
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("read actual bucket incarnation")
|
||||
}
|
||||
|
||||
async fn put(store: &ECStore, bucket: &str, object: &str) -> ObjectInfo {
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0xA5; 273]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("unsealed PUT must commit on all four disks")
|
||||
}
|
||||
|
||||
async fn disk_metadata(store: &ECStore, bucket: &str, object: &str) -> Vec<Vec<u8>> {
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
assert_eq!(disks.len(), 4);
|
||||
let mut metadata = Vec::new();
|
||||
for disk in disks {
|
||||
metadata.push(
|
||||
disk.expect("real fixture disk is online")
|
||||
.read_xl(bucket, object, false)
|
||||
.await
|
||||
.expect("read actual per-disk versions")
|
||||
.buf,
|
||||
);
|
||||
}
|
||||
metadata
|
||||
}
|
||||
|
||||
async fn assert_rejected_without_new_version(store: &ECStore, bucket: &str, object: &str, before: &[Vec<u8>]) {
|
||||
let result = store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0xBB; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_PUT_SEALED: later PUT must be rejected");
|
||||
assert_eq!(
|
||||
disk_metadata(store, bucket, object).await,
|
||||
before,
|
||||
"ARTIFACT_PUT_SEALED: no disk gains a version"
|
||||
);
|
||||
}
|
||||
|
||||
async fn tail_barrier_case(cancel_put_waiter: bool) {
|
||||
let root = tempfile::tempdir().expect("artifact tail fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-tail";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let tracker = rename_fanout_barrier::observe_tasks(&object);
|
||||
let tail = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let quorum = PutObjectCommitBarrier::install(bucket, &object, PutObjectCommitPause::AfterRenameQuorum);
|
||||
let handoff = PutObjectCommitBarrier::install(bucket, &object, PutObjectCommitPause::AfterRenameHandoff);
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let put_task = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
&put_object,
|
||||
&mut PutObjReader::from_vec(vec![0xA5; 273]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, tail.wait_until_paused())
|
||||
.await
|
||||
.expect("one real disk pauses before rename");
|
||||
quorum.wait_until_paused().await;
|
||||
if cancel_put_waiter {
|
||||
put_task.abort();
|
||||
}
|
||||
quorum.release();
|
||||
handoff.wait_until_paused().await;
|
||||
handoff.release();
|
||||
let result = tokio::time::timeout(DEADLINE, put_task)
|
||||
.await
|
||||
.expect("PUT waiter returns while its tail remains paused");
|
||||
if cancel_put_waiter {
|
||||
assert!(result.expect_err("cancelled PUT waiter").is_cancelled());
|
||||
} else {
|
||||
result
|
||||
.expect("PUT task joins")
|
||||
.expect("PUT quorum ACK succeeds before tail completion");
|
||||
}
|
||||
assert!(tracker.running() > 0, "real rename tail must still be live");
|
||||
let parallel_object = format!("{prefix}parallel.png");
|
||||
let parallel = tokio::time::timeout(DEADLINE, put(&store, bucket, ¶llel_object))
|
||||
.await
|
||||
.expect("ARTIFACT_PUT_SHARED_ADMISSION: another object in the same attempt completes while A is paused");
|
||||
let mut reader = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
store.get_object_reader(
|
||||
bucket,
|
||||
¶llel_object,
|
||||
None,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(parallel.version_id.expect("parallel PUT has a real version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("parallel object GET completes while A is paused")
|
||||
.expect("parallel object is readable");
|
||||
let mut bytes = Vec::new();
|
||||
tokio::time::timeout(DEADLINE, reader.stream.read_to_end(&mut bytes))
|
||||
.await
|
||||
.expect("parallel object body completes while A is paused")
|
||||
.expect("read parallel object body");
|
||||
assert_eq!(bytes, vec![0xA5; 273], "ARTIFACT_PUT_SHARED_ADMISSION: B publishes exact bytes");
|
||||
drop(reader);
|
||||
assert!(tracker.running() > 0, "A's physical tail remains paused throughout B's PUT and GET");
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
let attempt_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &record)
|
||||
.await
|
||||
.expect("resolve the actual PUT and seal coordination namespace");
|
||||
assert!(
|
||||
attempt_lock.get_write_lock(Duration::from_secs(1)).await.is_err(),
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: the real commit owner retains the attempt lock until its disk tail drains"
|
||||
);
|
||||
let seal_store = store.clone();
|
||||
let seal_prefix = prefix.clone();
|
||||
let mut seal =
|
||||
tokio::spawn(async move { seal_store.seal_artifact_attempt_puts(bucket, incarnation, &seal_prefix).await });
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(200), &mut seal).await.is_err(),
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: seal cannot finish while a real rename tail is paused"
|
||||
);
|
||||
assert!(tracker.running() > 0, "tail remains paused throughout the seal attempt");
|
||||
tail.release();
|
||||
tokio::time::timeout(DEADLINE, seal)
|
||||
.await
|
||||
.expect("seal resumes after tail release")
|
||||
.expect("seal task joins")
|
||||
.expect("seal persists");
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
while tracker.running() != 0 || ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("actual detached write owners drain");
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
})
|
||||
.await;
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_waits_for_quorum_ack_rename_tail() {
|
||||
tail_barrier_case(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_waits_after_put_waiter_cancellation() {
|
||||
tail_barrier_case(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_survives_reopen_and_is_attempt_scoped() {
|
||||
let root = tempfile::tempdir().expect("artifact durable seal directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-reopen";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("persist first seal");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("same seal is idempotent");
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
put(&store, bucket, &format!("{}other.png", attempt_prefix())).await;
|
||||
shutdown.cancel();
|
||||
assert!(
|
||||
ctx.wait_for_detached_mutations(DEADLINE).await,
|
||||
"old instance mutations drain before reopen"
|
||||
);
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
metadata_sys::reload_bucket_metadata(store.clone(), bucket)
|
||||
.await
|
||||
.expect("reload dedicated bucket after restart");
|
||||
assert_eq!(store.bucket_incarnation_id(bucket).await.expect("reopened incarnation"), incarnation);
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("persisted seal remains idempotent");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_put_seal_rejects_corrupt_and_wrong_incarnation_records() {
|
||||
super::run_large_stack_async_test("artifact-put-corrupt", artifact_put_corrupt_record_case);
|
||||
}
|
||||
|
||||
async fn artifact_put_corrupt_record_case() {
|
||||
let root = tempfile::tempdir().expect("artifact invalid seal record directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-corrupt";
|
||||
let prefix = attempt_prefix();
|
||||
let old_incarnation = create_bucket(&store, bucket).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, old_incarnation, &prefix)
|
||||
.await
|
||||
.expect("produce a real previous-generation seal");
|
||||
let old_record = com::read_config(store.clone(), &artifact_put_seal_record_path(bucket, old_incarnation, &prefix))
|
||||
.await
|
||||
.expect("read actual seal bytes");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete empty old-generation bucket");
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
assert_ne!(incarnation, old_incarnation);
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, old_incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"wrong expected incarnation cannot seal current bucket"
|
||||
);
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
for invalid_record in [old_record, Vec::new(), b"not-json".to_vec()] {
|
||||
com::save_config_with_opts(
|
||||
store.clone(),
|
||||
&record,
|
||||
invalid_record,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("persist actual invalid seal record on all disks");
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"invalid seal cannot be treated as absent or idempotent success"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_does_not_change_ordinary_bucket_puts() {
|
||||
let root = tempfile::tempdir().expect("ordinary bucket seal regression directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-ordinary";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create ordinary versioned bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("ordinary incarnation");
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let first = put(&store, bucket, &object).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"ordinary bucket cannot acquire artifact seal semantics"
|
||||
);
|
||||
let second = put(&store, bucket, &object).await;
|
||||
assert_ne!(first.version_id, second.version_id, "ordinary versioned PUT creates another version");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_put_seal_lock_survives_physical_publication_timeout() {
|
||||
// Match large ECStore fixtures while retaining the current-thread clock required by pause/advance.
|
||||
std::thread::Builder::new()
|
||||
.name("artifact-put-physical-timeout".to_owned())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("physical timeout test runtime should build");
|
||||
runtime.block_on(artifact_put_physical_timeout_case());
|
||||
})
|
||||
.expect("physical timeout test thread should spawn")
|
||||
.join()
|
||||
.expect("physical timeout test thread should complete");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn artifact_put_physical_timeout_case() {
|
||||
use crate::disk::{Disk, ReadOptions, STORAGE_FORMAT_FILE, os};
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
|
||||
(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false")),
|
||||
],
|
||||
async {
|
||||
let root = tempfile::tempdir().expect("artifact physical timeout directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-physical-timeout";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
let mut hooks = Vec::new();
|
||||
let mut entered = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut destinations = Vec::new();
|
||||
for disk in disks.iter().flatten() {
|
||||
let Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("physical timeout fixture requires local disks");
|
||||
};
|
||||
// Resolve through the mount lease; Linux may use an FD-based path.
|
||||
let destination = local
|
||||
.get_disk()
|
||||
.get_object_path_for_io(bucket, &object)
|
||||
.expect("real publication path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
hooks.push(os::prepared_publication_test_hooks::install(
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
// Dropping the sender also releases the syscall during panic unwinding.
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
));
|
||||
entered.push(entered_rx);
|
||||
releases.push(release_tx);
|
||||
destinations.push(destination);
|
||||
}
|
||||
assert_eq!(entered.len(), 4, "all four real disks must be online");
|
||||
let version = Uuid::new_v4();
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
&put_object,
|
||||
&mut PutObjReader::from_vec(vec![0xCC; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.to_string()),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
signals = futures::future::join_all(entered) => assert!(signals.into_iter().all(|signal| signal.is_ok())),
|
||||
_ = &mut replacement => panic!("PUT cannot finish before its physical publishers are paused"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all physical xl.meta publications reach the blocking syscall");
|
||||
// All disk wrapper timers exist, but their physical closures stay blocked.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(DEADLINE, replacement)
|
||||
.await
|
||||
.expect("PUT returns after disk wrapper timeouts")
|
||||
.expect("PUT task joins");
|
||||
assert!(result.is_err(), "four timed-out publishers cannot satisfy write quorum");
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
let attempt_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &record)
|
||||
.await
|
||||
.expect("actual attempt coordination namespace");
|
||||
let physical_owner_kept_lock = attempt_lock.get_write_lock(Duration::from_secs(1)).await.is_err();
|
||||
|
||||
// Drain physical syscalls before the regression assertion, including a RED run.
|
||||
drop(releases);
|
||||
for (disk, destination) in disks.iter().flatten().zip(&destinations) {
|
||||
drop(
|
||||
tokio::time::timeout(DEADLINE, os::acquire_rename_data_mutation_lease(&disk.path(), bucket, destination))
|
||||
.await
|
||||
.expect("released physical publisher drains"),
|
||||
);
|
||||
let published = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("timed-out physical writer actually publishes the exact new version");
|
||||
assert_eq!(published.size, 379);
|
||||
}
|
||||
assert!(
|
||||
physical_owner_kept_lock,
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: disk workers retain the attempt lock after the PUT coordinator returns its timeout"
|
||||
);
|
||||
tokio::time::timeout(DEADLINE, store.clone().seal_artifact_attempt_puts(bucket, incarnation, &prefix))
|
||||
.await
|
||||
.expect("seal resumes after physical owners drain")
|
||||
.expect("seal persists after timeout recovery");
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
shutdown.cancel();
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_rejects_mismatched_topology_and_preserves_ordinary_local_puts() {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
|
||||
const TOPOLOGY_ERROR: &str = "artifact PUT sealing requires local storage or a distributed Unix URL topology";
|
||||
let root = tempfile::tempdir().expect("artifact distributed topology directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let bucket = "artifact-put-distributed";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let ordinary = "artifact-put-distributed-ordinary";
|
||||
store
|
||||
.make_bucket(
|
||||
ordinary,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create ordinary versioned control bucket");
|
||||
let ordinary_incarnation = store.bucket_incarnation_id(ordinary).await.expect("ordinary identity");
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
let first = put(&store, ordinary, &object).await;
|
||||
|
||||
// Keep every physical disk online: the gate must inspect configured topology.
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
assert!(ctx.is_dist_erasure().await);
|
||||
let rejected = store
|
||||
.put_object(
|
||||
bucket,
|
||||
&object,
|
||||
&mut PutObjReader::from_vec(vec![0xDD; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("file endpoints with distributed lock mode must be rejected before publication");
|
||||
assert!(rejected.to_string().contains(TOPOLOGY_ERROR), "ARTIFACT_PUT_TOPOLOGY: {rejected}");
|
||||
assert_eq!(
|
||||
disk_metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"ARTIFACT_PUT_TOPOLOGY: no disk gains a version"
|
||||
);
|
||||
let rejected = store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect_err("mismatched topology cannot claim a durable PUT seal");
|
||||
assert!(rejected.to_string().contains(TOPOLOGY_ERROR), "ARTIFACT_PUT_TOPOLOGY: {rejected}");
|
||||
// This local fixture has no distributed lock clients; ordinary PUT retains its local regression check.
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let second = put(&store, ordinary, &object).await;
|
||||
assert_ne!(
|
||||
first.version_id, second.version_id,
|
||||
"ordinary local PUT stays available after artifact rejection"
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id(ordinary)
|
||||
.await
|
||||
.expect("ordinary identity remains available"),
|
||||
ordinary_incarnation
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("artifact identity remains available"),
|
||||
incarnation
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,689 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{artifact::artifact_version_retirement_record_path, metadata_sys},
|
||||
config::com,
|
||||
disk::{DiskAPI as _, DiskStore, RUSTFS_META_BUCKET, ReadOptions, STORAGE_FORMAT_FILE, os},
|
||||
error::Error,
|
||||
object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion},
|
||||
runtime::instance::InstanceContext,
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
heal::HealOperations,
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEADLINE: Duration = Duration::from_secs(30);
|
||||
const PAYLOAD_SIZE: usize = 1024 * 1024;
|
||||
|
||||
fn attempt_prefix() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn put_options() -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn heal_options() -> HealOpts {
|
||||
HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-retirement", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn create_bucket(store: &ECStore, bucket: &str, artifact: bool) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: artifact.then(|| "outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create real four-disk versioned bucket");
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("read actual bucket incarnation")
|
||||
}
|
||||
|
||||
async fn disks(store: &ECStore) -> Vec<DiskStore> {
|
||||
let disks = store.pools[0].disk_set[0]
|
||||
.disk_inventory()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|disk| disk.expect("fixture disk remains online"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(disks.len(), 4);
|
||||
disks
|
||||
}
|
||||
|
||||
fn object_path(disk: &DiskStore, bucket: &str, object: &str) -> PathBuf {
|
||||
disk.get_object_path_for_io_if_local(bucket, &encode_dir_object(object))
|
||||
.expect("fixture uses local disks")
|
||||
.expect("resolve object through held mount")
|
||||
}
|
||||
|
||||
async fn file_bytes(path: &Path) -> Option<Vec<u8>> {
|
||||
match tokio::fs::read(path).await {
|
||||
Ok(bytes) => Some(bytes),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
|
||||
Err(error) => panic!("read actual fixture file: {error}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn metadata(store: &ECStore, bucket: &str, object: &str) -> Vec<Option<Vec<u8>>> {
|
||||
let mut result = Vec::new();
|
||||
for disk in disks(store).await {
|
||||
let path = object_path(&disk, bucket, object).join(STORAGE_FORMAT_FILE);
|
||||
result.push(file_bytes(&path).await);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn put_payload(store: &ECStore, bucket: &str, object: &str, byte: u8, legacy: bool) -> Uuid {
|
||||
let mut options = put_options();
|
||||
if legacy {
|
||||
// Exercise genuine supported minority metadata recovery, without fabricated proofs.
|
||||
options.shard_integrity_write_mode = Some(ShardIntegrityWriteMode::Legacy);
|
||||
}
|
||||
let body = if object.ends_with('/') {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![byte; PAYLOAD_SIZE]
|
||||
};
|
||||
store
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(body), &options)
|
||||
.await
|
||||
.expect("commit exact payload and version to all four disks")
|
||||
.version_id
|
||||
.expect("artifact PUT returns a non-null version")
|
||||
}
|
||||
|
||||
async fn assert_payload(store: &ECStore, bucket: &str, object: &str, version: Uuid, byte: u8) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(version.to_string()),
|
||||
..put_options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retirement leaves the exact committed version readable");
|
||||
assert_eq!(reader.object_info.version_id, Some(version), "GET preserves exact version identity");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read exact committed bytes");
|
||||
let expected = if object.ends_with('/') {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![byte; PAYLOAD_SIZE]
|
||||
};
|
||||
assert_eq!(body, expected, "retirement does not delete or replace committed bytes");
|
||||
}
|
||||
|
||||
async fn remove_shard(store: &ECStore, bucket: &str, object: &str, version: Uuid) -> (DiskStore, PathBuf) {
|
||||
let disk = disks(store).await.remove(0);
|
||||
let info = disk
|
||||
.read_version("", bucket, &encode_dir_object(object), &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("read exact version before removing a real shard");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(&disk, bucket, object)
|
||||
.join(info.data_dir.expect("non-inline object has a data directory").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&part).await.expect("remove one real payload shard");
|
||||
assert!(!part.exists());
|
||||
(disk, part)
|
||||
}
|
||||
|
||||
async fn assert_healthy_version(store: &ECStore, bucket: &str, object: &str, version: Uuid) {
|
||||
for disk in disks(store).await {
|
||||
let info = disk
|
||||
.read_version("", bucket, &encode_dir_object(object), &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("repaired disk retains exact committed metadata");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
assert_eq!(info.size, i64::try_from(PAYLOAD_SIZE).expect("payload size fits i64"));
|
||||
let checked = disk
|
||||
.check_parts(bucket, &encode_dir_object(object), &info)
|
||||
.await
|
||||
.expect("verify genuine repaired shard");
|
||||
assert!(!checked.results.is_empty());
|
||||
assert!(
|
||||
checked
|
||||
.results
|
||||
.iter()
|
||||
.all(|result| *result == crate::disk::CHECK_PART_SUCCESS),
|
||||
"unretired version's real repaired shard passes integrity verification"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_policy_error(error: &Error) {
|
||||
assert!(
|
||||
!matches!(
|
||||
error,
|
||||
Error::FileNotFound | Error::FileVersionNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _)
|
||||
),
|
||||
"retirement policy refusal must not claim physical absence: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_repair_rejected(store: &ECStore, bucket: &str, object: &str, version: &str) -> Error {
|
||||
let result = HealOperations::heal_object(store, bucket, object, version, &heal_options()).await;
|
||||
let error = match result {
|
||||
Err(error) | Ok((_, Some(error))) => error,
|
||||
Ok((item, None)) => {
|
||||
panic!("ARTIFACT_RETIREMENT_NO_RESURRECTION: retired exact version was accepted for real repair: {item:?}")
|
||||
}
|
||||
};
|
||||
assert_policy_error(&error);
|
||||
// The proof-returning API must preserve the same refusal without converting it
|
||||
// into authoritative absence that a later destructive caller could consume.
|
||||
match store.heal_object_with_proof(bucket, object, version, &heal_options()).await {
|
||||
Err(error) => assert_policy_error(&error),
|
||||
Ok(result) => {
|
||||
assert!(result.absence.is_none(), "retirement never produces a physical absence proof");
|
||||
assert_policy_error(result.error.as_ref().expect("retired repair returns a policy error"));
|
||||
}
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
async fn assert_retired_repair_rejected(store: &ECStore, bucket: &str, object: &str, version: &str) {
|
||||
let error = assert_repair_rejected(store, bucket, object, version).await;
|
||||
assert!(
|
||||
error.to_string().contains("artifact object version is retired"),
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: expected retirement refusal, received {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn seal(store: &Arc<ECStore>, bucket: &str, incarnation: Uuid, prefix: &str) {
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, prefix)
|
||||
.await
|
||||
.expect("persist the attempt PUT seal before retirement");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_requires_seal_exact_identity_and_canonical_directory() {
|
||||
run_large_stack_async_test("artifact-retirement-identity", || async {
|
||||
let root = tempfile::tempdir().expect("retirement identity directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-identity";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let directory = format!("{prefix}folder/");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
let directory_version = put_payload(&store, bucket, &directory, 0, false).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"unsealed attempt cannot retire a version"
|
||||
);
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let wrong_prefix = attempt_prefix();
|
||||
seal(&store, bucket, incarnation, &wrong_prefix).await;
|
||||
for (expected, attempt, target, selected) in [
|
||||
(Uuid::new_v4(), prefix.clone(), object.clone(), version),
|
||||
(incarnation, wrong_prefix, object.clone(), version),
|
||||
(incarnation, prefix.clone(), object.clone(), Uuid::nil()),
|
||||
(incarnation, prefix.clone(), prefix.clone(), version),
|
||||
] {
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, expected, &attempt, &target, selected)
|
||||
.await
|
||||
.is_err(),
|
||||
"retirement requires the exact incarnation, attempt, inner object, and non-null version"
|
||||
);
|
||||
}
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
let ordinary = "artifact-retirement-ordinary";
|
||||
let ordinary_incarnation = create_bucket(&store, ordinary, false).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(ordinary, ordinary_incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"retirement is limited to dedicated artifact buckets"
|
||||
);
|
||||
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &directory, directory_version)
|
||||
.await
|
||||
.expect("retire logical directory identity");
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &prefix, &directory, directory_version);
|
||||
assert_eq!(
|
||||
record,
|
||||
artifact_version_retirement_record_path(
|
||||
bucket,
|
||||
incarnation,
|
||||
&prefix,
|
||||
&encode_dir_object(&directory),
|
||||
directory_version
|
||||
)
|
||||
);
|
||||
let bytes = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("read durable directory retirement");
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &encode_dir_object(&directory), directory_version)
|
||||
.await
|
||||
.expect("encoded directory retirement retry is idempotent");
|
||||
assert_eq!(com::read_config(store.clone(), &record).await.expect("read unchanged record"), bytes);
|
||||
assert_retired_repair_rejected(&store, bucket, &directory, &directory_version.to_string()).await;
|
||||
assert_retired_repair_rejected(&store, bucket, &encode_dir_object(&directory), &directory_version.to_string()).await;
|
||||
assert_payload(&store, bucket, &directory, directory_version, 0).await;
|
||||
// Rejected identity requests must not retire the valid regular object.
|
||||
let (_, missing) = remove_shard(&store, bucket, &object, version).await;
|
||||
let (_, error) = HealOperations::heal_object(store.as_ref(), bucket, &object, &version.to_string(), &heal_options())
|
||||
.await
|
||||
.expect("unretired version remains repairable after rejected retirement");
|
||||
assert!(error.is_none(), "unretired repair failed: {error:?}");
|
||||
assert!(missing.exists(), "unretired version's real shard is restored");
|
||||
assert_healthy_version(&store, bucket, &object, version).await;
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_rejects_explicit_and_latest_repair_but_preserves_other_version_after_reopen() {
|
||||
run_large_stack_async_test("artifact-retirement-versions", || async {
|
||||
let root = tempfile::tempdir().expect("retired version fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-versions";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let retained = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
let retired = put_payload(&store, bucket, &object, 0xB6, false).await;
|
||||
assert_ne!(retained, retired);
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
let mut physical_parts = Vec::new();
|
||||
for disk in disks(&store).await {
|
||||
for version in [retained, retired] {
|
||||
let info = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("snapshot exact version before retirement");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(&disk, bucket, &object)
|
||||
.join(info.data_dir.expect("payload data directory").to_string())
|
||||
.join("part.1");
|
||||
physical_parts.push((part.clone(), tokio::fs::read(part).await.expect("snapshot original shard bytes")));
|
||||
}
|
||||
}
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, retired)
|
||||
.await
|
||||
.expect("persist exact latest-version retirement");
|
||||
assert_eq!(
|
||||
metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"retirement itself does not edit object metadata"
|
||||
);
|
||||
for (part, bytes) in physical_parts {
|
||||
assert_eq!(
|
||||
tokio::fs::read(part).await.expect("retirement leaves every physical shard"),
|
||||
bytes,
|
||||
"retirement changes neither version's physical data"
|
||||
);
|
||||
}
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
let (_, retired_part) = remove_shard(&store, bucket, &object, retired).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
for selected in [retired.to_string(), String::new()] {
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &selected).await;
|
||||
assert!(
|
||||
!retired_part.exists(),
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: retired shard must stay absent"
|
||||
);
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before, "retired repair publishes no metadata");
|
||||
}
|
||||
let (_, retained_part) = remove_shard(&store, bucket, &object, retained).await;
|
||||
let (_, error) = HealOperations::heal_object(store.as_ref(), bucket, &object, &retained.to_string(), &heal_options())
|
||||
.await
|
||||
.expect("another version on the same object remains repairable");
|
||||
assert!(error.is_none(), "other-version repair failed: {error:?}");
|
||||
assert!(retained_part.exists(), "other version's actual shard is restored");
|
||||
assert_healthy_version(&store, bucket, &object, retained).await;
|
||||
assert!(!retired_part.exists(), "other-version repair must not restore the retired shard");
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(store.bucket_incarnation_id(bucket).await.expect("reopened incarnation"), incarnation);
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &retired.to_string()).await;
|
||||
assert!(!retired_part.exists(), "reopen cannot forget exact-version retirement");
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_blocks_minority_metadata_regeneration() {
|
||||
run_large_stack_async_test("artifact-retirement-regeneration", || async {
|
||||
let root = tempfile::tempdir().expect("minority metadata fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-regeneration";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, true).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.expect("persist version retirement before minority metadata damage");
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
let mut parts = Vec::new();
|
||||
for (index, disk) in disks(&store).await.iter().enumerate() {
|
||||
let info = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("read genuine legacy version before damage");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(disk, bucket, &object)
|
||||
.join(info.data_dir.expect("legacy payload has data directory").to_string())
|
||||
.join("part.1");
|
||||
parts.push((part.clone(), tokio::fs::read(&part).await.expect("snapshot real shard bytes")));
|
||||
if index > 0 {
|
||||
tokio::fs::remove_file(object_path(disk, bucket, &object).join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("leave exactly one genuine metadata copy");
|
||||
}
|
||||
}
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
assert_eq!(before.iter().filter(|entry| entry.is_some()).count(), 1);
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert_eq!(
|
||||
metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: minority regeneration must not publish retired metadata"
|
||||
);
|
||||
for (path, bytes) in parts {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("retired shard remains physically present"), bytes);
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_corrupt_or_wrong_identity_records_fail_closed() {
|
||||
run_large_stack_async_test("artifact-retirement-records", || async {
|
||||
let root = tempfile::tempdir().expect("corrupt retirement fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-records";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.expect("create genuine retirement record");
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &prefix, &object, version);
|
||||
let original = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("read genuine retirement JSON");
|
||||
let decoded: serde_json::Value = serde_json::from_slice(&original).expect("retirement record is JSON");
|
||||
let mut invalid = vec![Vec::new(), b"not-json".to_vec()];
|
||||
for (field, value) in [
|
||||
("format_version", serde_json::json!(2)),
|
||||
("bucket", serde_json::json!("different-bucket")),
|
||||
("base_prefix", serde_json::json!("other/")),
|
||||
("incarnation_id", serde_json::json!(Uuid::new_v4())),
|
||||
("attempt_prefix", serde_json::json!(attempt_prefix())),
|
||||
("object", serde_json::json!(format!("{prefix}other.png"))),
|
||||
("version_id", serde_json::json!(Uuid::new_v4())),
|
||||
("unexpected_field", serde_json::json!(true)),
|
||||
] {
|
||||
let mut changed = decoded.clone();
|
||||
changed
|
||||
.as_object_mut()
|
||||
.expect("strict identity record is an object")
|
||||
.insert(field.to_owned(), value);
|
||||
invalid.push(serde_json::to_vec(&changed).expect("encode invalid persisted identity"));
|
||||
}
|
||||
let (_, missing) = remove_shard(&store, bucket, &object, version).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
for bytes in invalid {
|
||||
// Inject corrupt persisted bytes through real guarded config I/O; the
|
||||
// fixture does not emulate the production retirement state machine.
|
||||
com::with_config_object_write_lock(store.clone(), record.clone(), {
|
||||
let store = store.clone();
|
||||
let record = record.clone();
|
||||
move |guard| async move { com::save_config_no_lock(store, &record, bytes, &guard).await }
|
||||
})
|
||||
.await
|
||||
.expect("hold matching record guard")
|
||||
.expect("persist invalid record on real disks");
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"corrupt or wrong-identity record cannot confirm idempotent retirement"
|
||||
);
|
||||
assert_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert!(!missing.exists(), "invalid record must not authorize real repair publication");
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_waits_for_cancelled_physical_repair_and_retries_cancelled_retirement() {
|
||||
run_large_stack_async_test("artifact-retirement-physical-tail", || async {
|
||||
let root = tempfile::tempdir().expect("physical retirement fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-physical-tail";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let (disk, part) = remove_shard(&store, bucket, &object, version).await;
|
||||
let destination = object_path(&disk, bucket, &object);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let repair_hook = os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
// Sender drop also releases physical work when an assertion unwinds.
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let repair_store = store.clone();
|
||||
let repair_object = object.clone();
|
||||
let mut repair = tokio::spawn(async move {
|
||||
HealOperations::heal_object(repair_store.as_ref(), bucket, &repair_object, &version.to_string(), &heal_options())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
entered = entered_rx => entered.expect("repair reaches real physical rename syscall"),
|
||||
result = &mut repair => panic!("repair returned before syscall pause: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("real repair worker is paused");
|
||||
repair.abort();
|
||||
assert!(repair.await.expect_err("cancel repair caller").is_cancelled());
|
||||
|
||||
let retire_store = store.clone();
|
||||
let retire_prefix = prefix.clone();
|
||||
let retire_object = object.clone();
|
||||
let mut retirement = tokio::spawn(async move {
|
||||
retire_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retire_prefix, &retire_object, version)
|
||||
.await
|
||||
});
|
||||
let blocked = tokio::time::timeout(Duration::from_millis(200), &mut retirement)
|
||||
.await
|
||||
.is_err();
|
||||
drop(release_tx);
|
||||
drop(
|
||||
tokio::time::timeout(DEADLINE, os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination))
|
||||
.await
|
||||
.expect("cancelled physical repair drains"),
|
||||
);
|
||||
drop(repair_hook);
|
||||
if blocked {
|
||||
tokio::time::timeout(DEADLINE, retirement)
|
||||
.await
|
||||
.expect("retirement resumes after real repair tail")
|
||||
.expect("retirement task joins")
|
||||
.expect("retirement persists");
|
||||
}
|
||||
assert!(
|
||||
blocked,
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: retirement must wait for cancelled repair's physical publication tail"
|
||||
);
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
// A cancelled repair may roll back its destination; retirement must stop
|
||||
// any later repair regardless of whether that first publication committed.
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
let part_before = file_bytes(&part).await;
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
assert_eq!(file_bytes(&part).await, part_before);
|
||||
|
||||
let second_prefix = attempt_prefix();
|
||||
let second_object = format!("{second_prefix}cancelled.bin");
|
||||
let second_version = put_payload(&store, bucket, &second_object, 0xB6, false).await;
|
||||
seal(&store, bucket, incarnation, &second_prefix).await;
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &second_prefix, &second_object, second_version);
|
||||
let record_disk = disks(&store).await.remove(0);
|
||||
let record_path = object_path(&record_disk, RUSTFS_META_BUCKET, &record).join(STORAGE_FORMAT_FILE);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _retirement_hook = os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
&record_path,
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let retire_store = store.clone();
|
||||
let retire_prefix = second_prefix.clone();
|
||||
let retire_object = second_object.clone();
|
||||
let mut retirement = tokio::spawn(async move {
|
||||
retire_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retire_prefix, &retire_object, second_version)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
entered = entered_rx => entered.expect("retirement reaches real record rename syscall"),
|
||||
result = &mut retirement => panic!("retirement returned before durable record publication: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retirement's real config publication is paused");
|
||||
retirement.abort();
|
||||
assert!(retirement.await.expect_err("cancel retirement caller").is_cancelled());
|
||||
let retry_store = store.clone();
|
||||
let retry_prefix = second_prefix.clone();
|
||||
let retry_object = second_object.clone();
|
||||
let mut retry = tokio::spawn(async move {
|
||||
retry_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retry_prefix, &retry_object, second_version)
|
||||
.await
|
||||
});
|
||||
let retry_blocked = tokio::time::timeout(Duration::from_millis(200), &mut retry).await.is_err();
|
||||
drop(release_tx);
|
||||
if retry_blocked {
|
||||
tokio::time::timeout(DEADLINE, retry)
|
||||
.await
|
||||
.expect("retry waits for original publication tail")
|
||||
.expect("retry task joins")
|
||||
.expect("cancelled retirement is confirmed idempotently");
|
||||
}
|
||||
assert!(retry_blocked, "retirement retry cannot pass a physically incomplete original publication");
|
||||
let persisted = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("cancelled retirement left durable record");
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &second_prefix, &second_object, second_version)
|
||||
.await
|
||||
.expect("idempotently confirm cancelled retirement");
|
||||
assert_eq!(com::read_config(store.clone(), &record).await.expect("read confirmed record"), persisted);
|
||||
assert_retired_repair_rejected(&store, bucket, &second_object, &second_version.to_string()).await;
|
||||
assert_payload(&store, bucket, &second_object, second_version, 0xB6).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,944 @@
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::metadata_sys,
|
||||
disk::DiskAPI as _,
|
||||
error::Error,
|
||||
object_api::{ObjectInfo, ObjectOptions, PutObjReader, WriteCommitGuard, WriteCompletion},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
multipart::{CompletePart, MultipartOperations as _},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{collections::HashMap, time::Duration};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn object_key() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/image.png", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn options() -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn body() -> PutObjReader {
|
||||
PutObjReader::from_vec(vec![0xA5; 273])
|
||||
}
|
||||
|
||||
fn unsupported<T>(result: Result<T, Error>) {
|
||||
let Err(error) = result else {
|
||||
panic!("ARTIFACT_WRITE_UNSUPPORTED: operation unexpectedly succeeded")
|
||||
};
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket does not support this write operation"),
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn bucket(store: &ECStore, name: &str, artifact: bool) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
name,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: artifact.then(|| "outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create real versioned fixture bucket");
|
||||
store.bucket_incarnation_id(name).await.expect("persisted incarnation")
|
||||
}
|
||||
|
||||
async fn snapshot(store: &ECStore, bucket: &str, object: &str) -> Vec<Vec<u8>> {
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
assert_eq!(disks.len(), 4);
|
||||
let mut versions = Vec::new();
|
||||
for disk in disks {
|
||||
versions.push(
|
||||
disk.expect("online fixture disk")
|
||||
.read_xl(bucket, object, false)
|
||||
.await
|
||||
.expect("physical version metadata")
|
||||
.buf,
|
||||
);
|
||||
}
|
||||
versions
|
||||
}
|
||||
|
||||
async fn copy_info(store: &ECStore, bucket: &str, object: &str) -> ObjectInfo {
|
||||
let mut info = store
|
||||
.get_object_info(bucket, object, &options())
|
||||
.await
|
||||
.expect("real source object");
|
||||
info.put_object_reader = Some(body());
|
||||
info
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_object_mutations_are_rejected() {
|
||||
run_large_stack_async_test("artifact-object-entrypoints", object_mutations_case);
|
||||
}
|
||||
|
||||
async fn object_mutations_case() {
|
||||
let root = tempfile::tempdir().expect("object entrypoint fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-object-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let artifact = "artifact-entrypoint-object";
|
||||
let ordinary = "artifact-entrypoint-control";
|
||||
bucket(&store, artifact, true).await;
|
||||
bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary owned artifact PUT is supported");
|
||||
store
|
||||
.put_object(ordinary, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary control object");
|
||||
let before = snapshot(&store, artifact, &object).await;
|
||||
|
||||
let mut source = copy_info(&store, ordinary, &object).await;
|
||||
unsupported(
|
||||
store
|
||||
.copy_object(ordinary, &object, artifact, &object, &mut source, &options(), &options())
|
||||
.await,
|
||||
);
|
||||
let mut source = copy_info(&store, artifact, &object).await;
|
||||
unsupported(
|
||||
store
|
||||
.copy_object(artifact, &object, artifact, &object, &mut source, &options(), &options())
|
||||
.await,
|
||||
);
|
||||
for opts in [
|
||||
ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
data_movement: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
preserve_ciphertext: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
metadata_chg: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
replication_status_writeback: Some(Box::default()),
|
||||
..options()
|
||||
},
|
||||
] {
|
||||
unsupported(store.put_object(artifact, &object, &mut body(), &opts).await);
|
||||
}
|
||||
unsupported(
|
||||
store
|
||||
.put_object_for_data_movement(
|
||||
artifact,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
..options()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
{
|
||||
let lock = store
|
||||
.new_ns_lock(artifact, &object)
|
||||
.await
|
||||
.expect("real borrowed target namespace");
|
||||
let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30))
|
||||
.await
|
||||
.expect("real borrowed write owner");
|
||||
let mut opts = options();
|
||||
opts.add_write_commit_guard(&guard);
|
||||
unsupported(store.put_object(artifact, &object, &mut body(), &opts).await);
|
||||
}
|
||||
unsupported(store.transition_object(artifact, &object, &options()).await);
|
||||
unsupported(store.clone().restore_transitioned_object(artifact, &object, &options()).await);
|
||||
unsupported(
|
||||
store
|
||||
.put_object_metadata(
|
||||
artifact,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("x-amz-meta-test".to_owned(), "changed".to_owned())])),
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
);
|
||||
unsupported(store.put_object_tags(artifact, &object, "kind=artifact", &options()).await);
|
||||
unsupported(store.delete_object_tags(artifact, &object, &options()).await);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: every disk retains the exact original versions"
|
||||
);
|
||||
|
||||
// An artifact source remains readable; only the destination write is classified.
|
||||
let mut source = copy_info(&store, artifact, &object).await;
|
||||
store
|
||||
.copy_object(artifact, &object, ordinary, "copied.png", &mut source, &options(), &options())
|
||||
.await
|
||||
.expect("copy into ordinary bucket stays supported");
|
||||
store
|
||||
.put_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary replication PUT stays supported");
|
||||
{
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(ordinary)
|
||||
.await
|
||||
.expect("ordinary incarnation");
|
||||
let lifecycle = std::sync::Arc::new(
|
||||
store
|
||||
.acquire_bucket_incarnation_fence(ordinary, incarnation)
|
||||
.await
|
||||
.expect("ordinary lifecycle owner"),
|
||||
);
|
||||
let lock = store.new_ns_lock(ordinary, &object).await.expect("ordinary target namespace");
|
||||
let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30))
|
||||
.await
|
||||
.expect("ordinary borrowed owner");
|
||||
let mut opts = options();
|
||||
lifecycle.attach_owner(&mut opts);
|
||||
opts.add_write_commit_guard(&guard);
|
||||
store
|
||||
.put_object(ordinary, &object, &mut body(), &opts)
|
||||
.await
|
||||
.expect("ordinary borrowed PUT stays supported");
|
||||
}
|
||||
store
|
||||
.put_object_metadata(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("x-amz-meta-test".to_owned(), "changed".to_owned())])),
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary metadata update stays supported");
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "kind=ordinary", &options())
|
||||
.await
|
||||
.expect("ordinary tag update stays supported");
|
||||
store
|
||||
.delete_object_tags(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary tag removal stays supported");
|
||||
assert_eq!(
|
||||
store
|
||||
.get_object_info(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary object remains readable")
|
||||
.size,
|
||||
273
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: reading an artifact source cannot mutate its versions"
|
||||
);
|
||||
let tier = "ARTIFACT-ENTRYPOINT-CONTROL";
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&ctx.tier_config_mgr(), tier).await;
|
||||
let current = store
|
||||
.get_object_info(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary source before transition");
|
||||
let exact = ObjectOptions {
|
||||
version_id: current.version_id.map(|version| version.to_string()),
|
||||
..options()
|
||||
};
|
||||
store
|
||||
.transition_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
transition: crate::bucket::lifecycle::lifecycle::TransitionOptions {
|
||||
status: rustfs_filemeta::TRANSITION_PENDING.to_owned(),
|
||||
tier: tier.to_owned(),
|
||||
etag: current.etag.expect("ordinary source etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: current.mod_time,
|
||||
..exact.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary transition stays supported");
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
let mut restore_transition = crate::bucket::lifecycle::lifecycle::TransitionOptions::default();
|
||||
restore_transition.restore_request.days = Some(1);
|
||||
store
|
||||
.clone()
|
||||
.restore_transitioned_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
transition: restore_transition,
|
||||
..exact
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary restore stays supported");
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_multipart_rejects_writes_but_keeps_legacy_cleanup() {
|
||||
run_large_stack_async_test("artifact-multipart-entrypoints", multipart_case);
|
||||
}
|
||||
|
||||
async fn multipart_case() {
|
||||
let root = tempfile::tempdir().expect("multipart entrypoint fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-multipart-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let artifact = "artifact-entrypoint-multipart";
|
||||
let ordinary = "artifact-multipart-control";
|
||||
let incarnation = bucket(&store, artifact, true).await;
|
||||
bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("seed the current artifact version");
|
||||
let before = snapshot(&store, artifact, &object).await;
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..options()
|
||||
};
|
||||
unsupported(store.new_multipart_upload(artifact, &object, &opts).await);
|
||||
// Seed actual legacy staging while holding the lifecycle owner required by the pool API.
|
||||
let lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock(artifact)
|
||||
.await
|
||||
.expect("legacy upload lifecycle owner");
|
||||
let mut staging_opts = opts.clone();
|
||||
staging_opts.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(artifact, &object, &staging_opts)
|
||||
.await
|
||||
.expect("real legacy upload staging");
|
||||
let part = store.pools[0]
|
||||
.put_object_part(artifact, &object, &upload.upload_id, 1, &mut body(), &staging_opts)
|
||||
.await
|
||||
.expect("real legacy staged part");
|
||||
drop(staging_opts);
|
||||
drop(lifecycle);
|
||||
let parts = vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag.clone(),
|
||||
..Default::default()
|
||||
}];
|
||||
unsupported(
|
||||
store
|
||||
.put_object_part(artifact, &object, &upload.upload_id, 2, &mut body(), &opts)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(artifact, &object, &upload.upload_id, parts.clone(), &opts)
|
||||
.await,
|
||||
);
|
||||
let movement = ObjectOptions {
|
||||
data_movement: true,
|
||||
part_number: Some(2),
|
||||
..opts.clone()
|
||||
};
|
||||
unsupported(
|
||||
store
|
||||
.handle_new_multipart_upload_with_pool_idx(artifact, &object, &movement, None)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.put_object_part_for_data_movement(0, artifact, &object, &upload.upload_id, &mut body(), &movement)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload_for_data_movement((0, None), artifact, &object, &upload.upload_id, parts, &movement)
|
||||
.await,
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: rejected multipart completion publishes no version"
|
||||
);
|
||||
let listing = store
|
||||
.list_multipart_uploads(artifact, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("list legacy uploads remains supported");
|
||||
assert_eq!(listing.uploads.len(), 1);
|
||||
assert_eq!(listing.uploads[0].upload_id, upload.upload_id);
|
||||
let staged = store
|
||||
.list_object_parts(artifact, &object, &upload.upload_id, None, 100, &opts)
|
||||
.await
|
||||
.expect("list legacy parts remains supported");
|
||||
assert_eq!(staged.parts.len(), 1);
|
||||
assert_eq!(staged.parts[0].etag, part.etag);
|
||||
store
|
||||
.abort_multipart_upload(artifact, &object, &upload.upload_id, &opts)
|
||||
.await
|
||||
.expect("abort legacy upload remains supported");
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(artifact, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("list after abort")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: legacy abort leaves published versions intact"
|
||||
);
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary multipart initialization");
|
||||
let part = store
|
||||
.put_object_part(ordinary, &object, &upload.upload_id, 1, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary part write");
|
||||
let completed = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
ordinary,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&options(),
|
||||
)
|
||||
.await
|
||||
.expect("ordinary multipart completion");
|
||||
assert_eq!(completed.size, 273);
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_classification_is_bound_to_bucket_incarnation() {
|
||||
run_large_stack_async_test("artifact-entrypoint-recreation", recreation_case);
|
||||
}
|
||||
|
||||
async fn recreation_case() {
|
||||
let root = tempfile::tempdir().expect("entrypoint recreation fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-entrypoint-recreation", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let name = "artifact-entrypoint-recreated";
|
||||
let old_incarnation = bucket(&store, name, false).await;
|
||||
let object = object_key();
|
||||
let barrier =
|
||||
crate::set_disk::PutObjectCommitBarrier::install(name, &object, crate::set_disk::PutObjectCommitPause::BeforeNamespace);
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
name,
|
||||
&put_object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::time::timeout(Duration::from_secs(30), store.delete_bucket(name, &DeleteBucketOptions::default()))
|
||||
.await
|
||||
.expect("staged old-generation upload does not prevent bucket deletion")
|
||||
.expect("delete old ordinary bucket");
|
||||
let new_incarnation = bucket(&store, name, true).await;
|
||||
assert_ne!(new_incarnation, old_incarnation);
|
||||
barrier.release();
|
||||
let error = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("stale staged PUT terminates")
|
||||
.expect("PUT task joins")
|
||||
.expect_err("classification of an ordinary bucket cannot authorize the new artifact bucket");
|
||||
assert!(
|
||||
matches!(error, Error::BucketNotFound(ref bucket) if bucket == name),
|
||||
"ARTIFACT_WRITE_INCARNATION: {error}"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let error = disk
|
||||
.expect("online fixture disk")
|
||||
.read_xl(name, &object, false)
|
||||
.await
|
||||
.err()
|
||||
.expect("no stale object metadata may be published");
|
||||
assert!(
|
||||
matches!(error, crate::disk::error::DiskError::FileNotFound),
|
||||
"ARTIFACT_WRITE_INCARNATION: {error}"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_lifecycle_fences_require_real_matching_owners() {
|
||||
run_large_stack_async_test("artifact-entrypoint-lifecycle-owner", lifecycle_owner_case);
|
||||
}
|
||||
|
||||
async fn lifecycle_owner_case() {
|
||||
let root = tempfile::tempdir().expect("lifecycle owner fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-lifecycle-owner", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let ordinary = "artifact-lifecycle-ordinary";
|
||||
let artifact = "artifact-lifecycle-dedicated";
|
||||
bucket(&store, ordinary, false).await;
|
||||
bucket(&store, artifact, true).await;
|
||||
let object = object_key();
|
||||
for name in [ordinary, artifact] {
|
||||
store
|
||||
.put_object(name, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("initial real object");
|
||||
}
|
||||
let before = snapshot(&store, ordinary, &object).await;
|
||||
let artifact_before = snapshot(&store, artifact, &object).await;
|
||||
let unrelated = store
|
||||
.new_ns_lock(ordinary, "unrelated-object")
|
||||
.await
|
||||
.expect("unrelated namespace");
|
||||
let unrelated_owner = unrelated
|
||||
.get_read_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("live unrelated owner");
|
||||
let mut foreign_fence = options();
|
||||
foreign_fence.add_bucket_lifecycle_lock_guard(&unrelated_owner);
|
||||
let lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock(ordinary)
|
||||
.await
|
||||
.expect("real lifecycle owner");
|
||||
let mut released_fence = options();
|
||||
released_fence.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
let borrowed_snapshot = store
|
||||
.object_lock_config_snapshot_under_lifecycle_fence(
|
||||
ordinary,
|
||||
released_fence
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.expect("borrowed lifecycle fence"),
|
||||
)
|
||||
.await
|
||||
.expect("real snapshot retaining only its metadata lock");
|
||||
drop(lifecycle);
|
||||
let mut snapshot_without_owner = options();
|
||||
borrowed_snapshot.add_lock_fences(&mut snapshot_without_owner);
|
||||
snapshot_without_owner.object_lock_config_snapshot = Some(borrowed_snapshot);
|
||||
|
||||
for opts in [foreign_fence, released_fence, snapshot_without_owner] {
|
||||
let mut source = copy_info(&store, ordinary, &object).await;
|
||||
for result in [
|
||||
store
|
||||
.put_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store
|
||||
.copy_object(ordinary, &object, ordinary, &object, &mut source, &options(), &opts)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "owner=fake", &opts)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store.put_object_metadata(ordinary, &object, &opts).await.map(|_| ()),
|
||||
store.clone().restore_transitioned_object(ordinary, &object, &opts).await,
|
||||
store.new_multipart_upload(ordinary, &object, &opts).await.map(|_| ()),
|
||||
] {
|
||||
let error = result.expect_err("a fence without its matching lifecycle owner cannot authorize a write");
|
||||
assert!(
|
||||
error.to_string().contains("write requires an owned bucket lifecycle fence"),
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: {error}"
|
||||
);
|
||||
}
|
||||
let error = store
|
||||
.put_object(artifact, &object, &mut body(), &opts)
|
||||
.await
|
||||
.expect_err("ordinary artifact PUT also requires the lifecycle owner");
|
||||
assert!(
|
||||
error.to_string().contains("write requires an owned bucket lifecycle fence"),
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, ordinary, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: ordinary metadata unchanged"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
artifact_before,
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: artifact versions unchanged"
|
||||
);
|
||||
}
|
||||
drop(unrelated_owner);
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(ordinary, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("inspect rejected multipart creation")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let owned_snapshot = store
|
||||
.object_lock_config_snapshot(ordinary)
|
||||
.await
|
||||
.expect("real lifecycle-owning snapshot");
|
||||
let mut owned = options();
|
||||
owned_snapshot.add_lock_fences(&mut owned);
|
||||
owned.object_lock_config_snapshot = Some(owned_snapshot.clone());
|
||||
let lifecycle_namespace = store
|
||||
.new_ns_lock(ordinary, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||
.await
|
||||
.expect("actual bucket deletion namespace");
|
||||
assert!(
|
||||
lifecycle_namespace.get_write_lock(Duration::from_millis(100)).await.is_err(),
|
||||
"matching snapshot actually blocks the bucket deletion lock"
|
||||
);
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "owner=real", &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary tags");
|
||||
let upload = store
|
||||
.new_multipart_upload(ordinary, &object, &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary multipart creation");
|
||||
store
|
||||
.abort_multipart_upload(ordinary, &object, &upload.upload_id, &owned)
|
||||
.await
|
||||
.expect("clean up supported multipart upload");
|
||||
drop(owned);
|
||||
drop(owned_snapshot);
|
||||
let artifact_snapshot = store
|
||||
.object_lock_config_snapshot(artifact)
|
||||
.await
|
||||
.expect("artifact lifecycle-owning snapshot");
|
||||
let mut owned = options();
|
||||
artifact_snapshot.add_lock_fences(&mut owned);
|
||||
owned.object_lock_config_snapshot = Some(artifact_snapshot);
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary artifact PUT");
|
||||
assert_ne!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
artifact_before,
|
||||
"supported artifact PUT publishes a new version"
|
||||
);
|
||||
drop(owned);
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
// Include directories as well as bytes: a rejected delete must not even stage
|
||||
// a rollback directory before another layer can refuse the request.
|
||||
async fn delete_physical_snapshot(
|
||||
store: &ECStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Vec<std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>> {
|
||||
let mut snapshots = Vec::new();
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let disk = disk.expect("online delete fixture disk");
|
||||
let root = disk.path().join(bucket).join(object);
|
||||
let mut entries = std::collections::BTreeMap::new();
|
||||
let mut pending = vec![root.clone()];
|
||||
while let Some(directory) = pending.pop() {
|
||||
let mut children = tokio::fs::read_dir(&directory)
|
||||
.await
|
||||
.expect("read physical artifact directory");
|
||||
while let Some(child) = children.next_entry().await.expect("read physical artifact entry") {
|
||||
let path = child.path();
|
||||
let relative = path.strip_prefix(&root).expect("fixture descendant").to_owned();
|
||||
if child.file_type().await.expect("physical entry type").is_dir() {
|
||||
entries.insert(relative, None);
|
||||
pending.push(path);
|
||||
} else {
|
||||
entries.insert(relative, Some(tokio::fs::read(path).await.expect("physical artifact bytes")));
|
||||
}
|
||||
}
|
||||
}
|
||||
snapshots.push(entries);
|
||||
}
|
||||
snapshots
|
||||
}
|
||||
|
||||
async fn seed_delete_versions(store: &ECStore, bucket: &str, object: &str, incarnation: Uuid) -> (Uuid, Uuid) {
|
||||
let first = store
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0xA5; 2 * 1024 * 1024]), &options())
|
||||
.await
|
||||
.expect("publish actual external artifact shards")
|
||||
.version_id
|
||||
.expect("versioned artifact identity");
|
||||
store
|
||||
.put_object(bucket, object, &mut body(), &options())
|
||||
.await
|
||||
.expect("publish another readable artifact version");
|
||||
let marker = Uuid::new_v4();
|
||||
let mut fi = rustfs_filemeta::FileInfo {
|
||||
name: object.to_owned(),
|
||||
version_id: Some(marker),
|
||||
deleted: true,
|
||||
mod_time: Some(time::OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
fi.set_delete_marker_incarnation(incarnation);
|
||||
// Seed a pre-existing marker through the physical fixture API. Ordinary
|
||||
// managed deletion must never be needed to construct its own refusal case.
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
disk.expect("online marker fixture disk")
|
||||
.write_metadata("", bucket, object, fi.clone())
|
||||
.await
|
||||
.expect("persist pre-existing delete marker");
|
||||
}
|
||||
(first, marker)
|
||||
}
|
||||
|
||||
fn delete_variants(version: Uuid, marker: Uuid) -> Vec<ObjectOptions> {
|
||||
vec![
|
||||
options(),
|
||||
ObjectOptions {
|
||||
version_id: Some(version.to_string()),
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: Some(marker.to_string()),
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
delete_marker: true,
|
||||
replication_request: true,
|
||||
delete_replication: Some(crate::bucket::replication::ReplicationState::default()),
|
||||
..options()
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_delete_entrypoints_store_rejects_single_and_batch_before_mutation() {
|
||||
run_large_stack_async_test("artifact-delete-store", || delete_entrypoints_case(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_delete_entrypoints_set_disks_rejects_direct_mutations() {
|
||||
run_large_stack_async_test("artifact-delete-set-disks", || delete_entrypoints_case(true));
|
||||
}
|
||||
|
||||
async fn delete_entrypoints_case(direct: bool) {
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
let root = tempfile::tempdir().expect("delete entrypoint fixture root");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-delete-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let managed = "artifact-delete-managed";
|
||||
let ordinary = "artifact-delete-control";
|
||||
let incarnation = bucket(&store, managed, true).await;
|
||||
let ordinary_incarnation = bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
let (version, marker) = seed_delete_versions(&store, managed, &object, incarnation).await;
|
||||
let (control_version, control_marker) = seed_delete_versions(&store, ordinary, &object, ordinary_incarnation).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let before = delete_physical_snapshot(&store, managed, &object).await;
|
||||
assert!(
|
||||
before.iter().all(|disk| disk.keys().any(|path| path.ends_with("part.1"))),
|
||||
"fixture must include external shards"
|
||||
);
|
||||
|
||||
for sealed in [false, true] {
|
||||
if sealed {
|
||||
let attempt = object.rsplit_once('/').expect("attempt/object fixture").0.to_owned() + "/";
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(managed, incarnation, &attempt)
|
||||
.await
|
||||
.expect("seal ordinary PUT domain");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_repairs(managed, incarnation, &attempt)
|
||||
.await
|
||||
.expect("seal repair domain");
|
||||
}
|
||||
for opts in delete_variants(version, marker) {
|
||||
let result = if direct {
|
||||
set.delete_object(managed, &object, opts).await
|
||||
} else {
|
||||
// Hold the object lock to distinguish the ECStore guard from a
|
||||
// later SetDisks refusal, without removing that second defense.
|
||||
let lock = store.new_ns_lock(managed, &object).await.expect("delete namespace");
|
||||
let _held = lock
|
||||
.get_write_lock(Duration::from_secs(5))
|
||||
.await
|
||||
.expect("hold delete namespace");
|
||||
tokio::time::timeout(Duration::from_secs(1), store.delete_object(managed, &object, opts))
|
||||
.await
|
||||
.expect("managed rejection precedes object locking")
|
||||
};
|
||||
unsupported(result);
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
}
|
||||
let targets = vec![
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(marker),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker_replication_status: Some("REPLICA".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let (deleted, errors) = if direct {
|
||||
set.delete_objects(managed, targets.clone(), options()).await
|
||||
} else {
|
||||
let lock = store.new_ns_lock(managed, &object).await.expect("batch namespace");
|
||||
let _held = lock
|
||||
.get_write_lock(Duration::from_secs(5))
|
||||
.await
|
||||
.expect("hold batch namespace");
|
||||
tokio::time::timeout(Duration::from_secs(1), store.delete_objects(managed, targets.clone(), options()))
|
||||
.await
|
||||
.expect("managed batch rejection precedes object locking")
|
||||
};
|
||||
assert_eq!(deleted.len(), targets.len());
|
||||
assert_eq!(errors.len(), targets.len());
|
||||
for error in errors {
|
||||
unsupported::<()>(Err(error.expect("each managed delete retains its own refusal")));
|
||||
}
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
let request = rustfs_filemeta::FileInfo {
|
||||
version_id: Some(version),
|
||||
name: object.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
unsupported(if direct {
|
||||
set.delete_object_version(managed, &object, &request, false).await
|
||||
} else {
|
||||
store.delete_object_version(managed, &object, &request, false).await
|
||||
});
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
}
|
||||
|
||||
let control_before = delete_physical_snapshot(&store, ordinary, &object).await;
|
||||
let mut control_opts = options();
|
||||
if direct {
|
||||
// An unbound policy snapshot remains supported, but is not lifecycle
|
||||
// authority. The direct entry point must acquire its own real owner.
|
||||
control_opts.object_lock_config_snapshot = Some(std::sync::Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
|
||||
metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
)));
|
||||
}
|
||||
let created = if direct {
|
||||
set.delete_object(ordinary, &object, control_opts.clone()).await
|
||||
} else {
|
||||
store.delete_object(ordinary, &object, control_opts.clone()).await
|
||||
}
|
||||
.expect("ordinary bucket actually creates a delete marker");
|
||||
assert!(created.delete_marker);
|
||||
assert_ne!(delete_physical_snapshot(&store, ordinary, &object).await, control_before);
|
||||
let control_targets = vec![
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(control_version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(control_marker),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let (_, errors) = if direct {
|
||||
set.delete_objects(ordinary, control_targets, control_opts).await
|
||||
} else {
|
||||
store.delete_objects(ordinary, control_targets, control_opts).await
|
||||
};
|
||||
assert!(errors.iter().all(Option::is_none), "ordinary batch deletion must commit: {errors:?}");
|
||||
for disk in set.disk_inventory().await {
|
||||
let disk = disk.expect("online control disk");
|
||||
for removed in [control_version, control_marker] {
|
||||
assert!(
|
||||
disk.read_version(ordinary, ordinary, &object, &removed.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"ordinary deletion removed the exact physical version"
|
||||
);
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -86,6 +86,160 @@ pub async fn connect_load_init_formats(
|
||||
pub(crate) struct LoadedFormat {
|
||||
pub(crate) format: FormatV3,
|
||||
pub(crate) pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority,
|
||||
pub(crate) pending_disks: Vec<PendingFormatDisk>,
|
||||
}
|
||||
|
||||
pub(crate) struct PendingFormatDisk {
|
||||
disk: DiskStore,
|
||||
}
|
||||
|
||||
impl PendingFormatDisk {
|
||||
/// Reconcile only after bootstrap metadata has succeeded, before Ready.
|
||||
pub(crate) async fn finalize(
|
||||
&self,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
sets: &[Arc<crate::set_disk::SetDisks>],
|
||||
) -> Result<()> {
|
||||
let endpoint = self.disk.endpoint();
|
||||
let pool_index = usize::try_from(endpoint.pool_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set_index = usize::try_from(endpoint.set_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let disk_index = usize::try_from(endpoint.disk_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set = sets
|
||||
.iter()
|
||||
.find(|set| set.pool_index == pool_index && set.set_index == set_index)
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if set.set_endpoints.get(disk_index) != Some(&endpoint) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
// Capture current handles, including a reconnect that completed since
|
||||
// format selection. No registry guard is held during format I/O.
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let receivers = registry.read().await;
|
||||
let receiver = if endpoint.is_local {
|
||||
receivers.get(&endpoint.to_string()).and_then(Clone::clone)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let selected = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.get(disk_index)
|
||||
.cloned()
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if endpoint.is_local && receiver.is_none() {
|
||||
// An earlier successful pass may already have quarantined this
|
||||
// slot before a different slot raced with reconnect and retried.
|
||||
let topology = instance_ctx.local_disk_set_drives();
|
||||
let local_slots = topology.read().await;
|
||||
let local_slot = local_slots
|
||||
.get(pool_index)
|
||||
.and_then(|sets| sets.get(set_index))
|
||||
.and_then(|disks| disks.get(disk_index))
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
return if matches!(receivers.get(&endpoint.to_string()), Some(None)) && local_slot.is_none() && selected.is_none() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::other("format reconciliation receiver changed"))
|
||||
};
|
||||
}
|
||||
drop(receivers);
|
||||
let disk = selected.as_ref().unwrap_or(&self.disk);
|
||||
let observation = load_format_erasure(receiver.as_ref().unwrap_or(disk), false).await;
|
||||
self.publish_observation(instance_ctx, set, selected.as_ref(), receiver.as_ref(), observation)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn publish_observation(
|
||||
&self,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
set: &crate::set_disk::SetDisks,
|
||||
selected: Option<&DiskStore>,
|
||||
receiver: Option<&DiskStore>,
|
||||
observation: disk::error::Result<FormatV3>,
|
||||
) -> Result<()> {
|
||||
let endpoint = self.disk.endpoint();
|
||||
let pool_index = usize::try_from(endpoint.pool_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set_index = usize::try_from(endpoint.set_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let disk_index = usize::try_from(endpoint.disk_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
if set.pool_index != pool_index || set.set_index != set_index || set.set_endpoints.get(disk_index) != Some(&endpoint) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
let valid = observation.as_ref().is_ok_and(|format| {
|
||||
set.format.check_other(format).is_ok()
|
||||
&& set.format.find_disk_index_by_disk_id(format.erasure.this).ok() == Some((set_index, disk_index))
|
||||
});
|
||||
let disk_id = if valid {
|
||||
Some(observation.map_err(Error::from)?.erasure.this)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let path = endpoint.to_string();
|
||||
// Local sets must share the registered receiver's health and I/O
|
||||
// admission state, just as Sets::new_with_instance_ctx does.
|
||||
let disk = receiver.unwrap_or_else(|| selected.unwrap_or(&self.disk));
|
||||
if disk.endpoint() != endpoint
|
||||
|| endpoint.is_local != receiver.is_some()
|
||||
|| receiver.is_some_and(|receiver| receiver.endpoint() != endpoint)
|
||||
{
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
|
||||
// All format I/O is complete. Publish under registry -> ID map -> local
|
||||
// topology -> set slot, rechecking handle identity before any mutation.
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let mut receivers = registry.write().await;
|
||||
let id_map = instance_ctx.local_disk_id_map();
|
||||
let mut ids = id_map.write().await;
|
||||
let topology = instance_ctx.local_disk_set_drives();
|
||||
let mut local_slots = topology.write().await;
|
||||
let mut slots = set.disks.write().await;
|
||||
let slot = slots.get_mut(disk_index).ok_or(Error::CorruptedFormat)?;
|
||||
let unchanged = match (slot.as_ref(), selected) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(observed)) => Arc::ptr_eq(current, observed),
|
||||
_ => false,
|
||||
};
|
||||
if !unchanged {
|
||||
return Err(Error::other("format reconciliation slot changed"));
|
||||
}
|
||||
if let Some(receiver) = receiver {
|
||||
if !receivers
|
||||
.get(&path)
|
||||
.and_then(Option::as_ref)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, receiver))
|
||||
{
|
||||
return Err(Error::other("format reconciliation receiver changed"));
|
||||
}
|
||||
let local_slot = local_slots
|
||||
.get_mut(pool_index)
|
||||
.and_then(|sets| sets.get_mut(set_index))
|
||||
.and_then(|disks| disks.get_mut(disk_index))
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if !local_slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, receiver)) {
|
||||
return Err(Error::other("format reconciliation local slot changed"));
|
||||
}
|
||||
// Local ID setters only update memory, without filesystem I/O.
|
||||
receiver.set_disk_id_state(disk_id).await?;
|
||||
ids.retain(|_, registered_path| registered_path != &path);
|
||||
if let Some(id) = disk_id {
|
||||
ids.insert(id, path.clone());
|
||||
} else {
|
||||
receivers.insert(path, None);
|
||||
*local_slot = None;
|
||||
}
|
||||
}
|
||||
if receiver.is_none() {
|
||||
disk.set_disk_id_state(disk_id).await?;
|
||||
}
|
||||
if valid {
|
||||
disk.enable_health_check();
|
||||
*slot = Some(disk.clone());
|
||||
} else {
|
||||
*slot = None;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
@@ -137,13 +291,15 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &members, set_drive_count).await?;
|
||||
let pending_disks =
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &members, set_drive_count, Some(&errs)).await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
pending_disks,
|
||||
});
|
||||
}
|
||||
let resumable_partial_migration = formats_present > 0
|
||||
@@ -158,10 +314,11 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
|
||||
info!("Migrated format from MinIO config");
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count, None).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: *format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::LegacyAdoption,
|
||||
pending_disks: Vec::new(),
|
||||
});
|
||||
}
|
||||
Ok(LegacyFormatOutcome::Incompatible) => {
|
||||
@@ -182,6 +339,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
|
||||
pending_disks: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -206,11 +364,13 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
None => select_format_erasure_in_quorum(&formats, 0)?,
|
||||
};
|
||||
check_format_erasure_value_for_topology(&fm, formats.len(), set_drive_count)?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &fm, &quorum_members, set_drive_count).await?;
|
||||
let pending_disks =
|
||||
retain_format_quorum_members(instance_ctx, disks, &fm, &quorum_members, set_drive_count, Some(&errs)).await?;
|
||||
|
||||
Ok(LoadedFormat {
|
||||
format: fm,
|
||||
pool_meta_bootstrap_authority: verified_legacy_adoption_source(disks, &formats, set_count, set_drive_count).await?,
|
||||
pending_disks,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -239,7 +399,8 @@ async fn retain_format_quorum_members(
|
||||
format: &FormatV3,
|
||||
quorum_members: &[bool],
|
||||
set_drive_count: usize,
|
||||
) -> Result<()> {
|
||||
observed_errors: Option<&[Option<DiskError>]>,
|
||||
) -> Result<Vec<PendingFormatDisk>> {
|
||||
if set_drive_count == 0 || quorum_members.len() != disks.len() {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
@@ -279,10 +440,31 @@ async fn retain_format_quorum_members(
|
||||
let mut local_pool_endpoints = Vec::new();
|
||||
let mut selected_local_disk_ids = Vec::new();
|
||||
let mut quarantined_endpoints = Vec::new();
|
||||
let mut pending_disks = Vec::new();
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let receivers = registry.read().await;
|
||||
for (index, ((disk, endpoint), belongs_to_quorum)) in disks.iter().zip(&endpoints).zip(quorum_members).enumerate() {
|
||||
let pending = !belongs_to_quorum
|
||||
&& matches!(
|
||||
observed_errors.and_then(|errors| errors.get(index)),
|
||||
Some(Some(DiskError::UnformattedDisk))
|
||||
);
|
||||
let receiver = endpoint
|
||||
.as_ref()
|
||||
.filter(|endpoint| endpoint.is_local)
|
||||
.and_then(|endpoint| receivers.get(&endpoint.to_string()))
|
||||
.and_then(Clone::clone);
|
||||
// A first writer may still be publishing this slot's format. Keep its
|
||||
// receiver reachable, but do not count it as a format quorum member.
|
||||
let defer = pending && disk.as_ref().is_some_and(|disk| !disk.is_local() || receiver.is_some());
|
||||
if defer {
|
||||
pending_disks.push(PendingFormatDisk {
|
||||
disk: disk.as_ref().ok_or(Error::CorruptedFormat)?.clone(),
|
||||
});
|
||||
}
|
||||
if let Some(endpoint) = endpoint.as_ref().filter(|endpoint| endpoint.is_local) {
|
||||
local_pool_endpoints.push(endpoint.to_string());
|
||||
if !belongs_to_quorum {
|
||||
if !belongs_to_quorum && !defer {
|
||||
quarantined_endpoints.push(endpoint.clone());
|
||||
}
|
||||
}
|
||||
@@ -302,6 +484,7 @@ async fn retain_format_quorum_members(
|
||||
selected_local_disk_ids.push((disk_id, disk.endpoint().to_string()));
|
||||
}
|
||||
}
|
||||
drop(receivers);
|
||||
quarantine_local_disks(instance_ctx, &quarantined_endpoints).await?;
|
||||
|
||||
for (disk, disk_id) in disks.iter().zip(member_disk_ids) {
|
||||
@@ -315,7 +498,7 @@ async fn retain_format_quorum_members(
|
||||
*disk = None;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
Ok(pending_disks)
|
||||
}
|
||||
|
||||
pub fn should_init_erasure_disks(errs: &[Option<DiskError>]) -> bool {
|
||||
@@ -369,7 +552,7 @@ async fn init_format_erasure(
|
||||
write_format_file_all(disks, &fms).await?;
|
||||
let format = get_format_erasure_in_quorum(&fms, 0)?;
|
||||
let quorum_members = vec![true; disks.len()];
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count, None).await?;
|
||||
Ok(format)
|
||||
}
|
||||
|
||||
@@ -976,6 +1159,314 @@ mod tests {
|
||||
(temp_dir, disks)
|
||||
}
|
||||
|
||||
struct PendingFormatFixture {
|
||||
ctx: Arc<InstanceContext>,
|
||||
set: Arc<crate::set_disk::SetDisks>,
|
||||
pending: PendingFormatDisk,
|
||||
receiver: DiskStore,
|
||||
format: FormatV3,
|
||||
_temp: tempfile::TempDir,
|
||||
}
|
||||
|
||||
async fn pending_format_fixture() -> PendingFormatFixture {
|
||||
let (temp, mut disks) = local_disks(4).await;
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let format = FormatV3::new(1, 4);
|
||||
let endpoints = disks.iter().flatten().map(|disk| disk.endpoint()).collect::<Vec<_>>();
|
||||
// Bootstrap receivers are distinct from the subsequently opened set handles.
|
||||
let (receivers, errors) = init_disks(
|
||||
&Endpoints::from(endpoints.clone()),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none));
|
||||
record_local_disks(&ctx, receivers.iter().flatten().cloned().collect()).await;
|
||||
*ctx.local_disk_set_drives().write().await = vec![vec![receivers.clone()]];
|
||||
for (index, disk) in disks.iter().enumerate().take(3) {
|
||||
let mut member = format.clone();
|
||||
member.erasure.this = format.erasure.sets[0][index];
|
||||
save_format_file(disk, &Some(member))
|
||||
.await
|
||||
.expect("first three formats should publish");
|
||||
}
|
||||
let receiver = receivers[3].as_ref().expect("pending receiver should exist").clone();
|
||||
let mut last = format.clone();
|
||||
last.erasure.this = format.erasure.sets[0][3];
|
||||
let bootstrap = crate::store::peer::BootstrapLocalTarget::new(ctx.clone());
|
||||
bootstrap
|
||||
.begin_legacy_local_file_write(&receiver, RUSTFS_META_BUCKET, None)
|
||||
.await
|
||||
.expect("pending format writer should be admitted")
|
||||
.write_all("pending-format", last.to_json().expect("format should encode").into_bytes().into())
|
||||
.await
|
||||
.expect("last format temporary file should be durable");
|
||||
let mut loaded = connect_load_init_formats_with_instance_ctx(&ctx, false, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("three existing formats should supply read quorum");
|
||||
assert!(matches!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None));
|
||||
assert_eq!(loaded.pending_disks.len(), 1);
|
||||
assert!(disks[3].is_none(), "unformatted slot must not count as a member");
|
||||
let set = crate::set_disk::SetDisks::new_with_instance_ctx(
|
||||
"pending-format-test".to_string(),
|
||||
Arc::new(tokio::sync::RwLock::new(disks)),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
loaded.format,
|
||||
Vec::new(),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
PendingFormatFixture {
|
||||
ctx,
|
||||
set,
|
||||
pending: loaded.pending_disks.remove(0),
|
||||
receiver,
|
||||
format,
|
||||
_temp: temp,
|
||||
}
|
||||
}
|
||||
|
||||
async fn publish_pending_format(fixture: &PendingFormatFixture) {
|
||||
let bootstrap = crate::store::peer::BootstrapLocalTarget::new(fixture.ctx.clone());
|
||||
// Resolve the endpoint again, just as the writer's next RPC does. A
|
||||
// retained Arc alone would bypass the receiver quarantine regression.
|
||||
let resolved = bootstrap
|
||||
.local_mutation_disk(&fixture.receiver.endpoint().to_string())
|
||||
.await
|
||||
.expect("pending receiver must remain endpoint-addressable");
|
||||
assert!(Arc::ptr_eq(&resolved, &fixture.receiver));
|
||||
bootstrap
|
||||
.begin_legacy_local_rename_file(
|
||||
&fixture.receiver.endpoint().to_string(),
|
||||
(RUSTFS_META_BUCKET, "pending-format"),
|
||||
(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("writer's next rename should still be admitted")
|
||||
.commit()
|
||||
.await
|
||||
.expect("writer should complete the last format");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_preserves_pending_bootstrap_receiver_and_restores_completed_slot() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
publish_pending_format(&fixture).await;
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("completed format should reconcile before Ready");
|
||||
let slots = fixture.set.disks.read().await;
|
||||
assert_eq!(slots.iter().flatten().count(), 4);
|
||||
assert!(Arc::ptr_eq(
|
||||
slots[3].as_ref().expect("restored local member should exist"),
|
||||
&fixture.receiver
|
||||
));
|
||||
for (index, disk) in slots.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("all four members should be restored");
|
||||
let loaded = load_format_erasure(disk, false)
|
||||
.await
|
||||
.expect("restored member format should read");
|
||||
assert_eq!(loaded.shared_identity(), fixture.format.shared_identity());
|
||||
assert_eq!(loaded.erasure.this, fixture.format.erasure.sets[0][index]);
|
||||
}
|
||||
let id = fixture.format.erasure.sets[0][3];
|
||||
assert_eq!(
|
||||
slots[3]
|
||||
.as_ref()
|
||||
.expect("last member should exist")
|
||||
.get_disk_id()
|
||||
.await
|
||||
.expect("set disk ID should read"),
|
||||
Some(id)
|
||||
);
|
||||
assert_eq!(fixture.receiver.get_disk_id().await.expect("receiver ID should read"), Some(id));
|
||||
assert_eq!(
|
||||
fixture.ctx.local_disk_id_map().read().await.get(&id),
|
||||
Some(&fixture.receiver.endpoint().to_string())
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.ctx.local_disk_set_drives().read().await[0][0][3]
|
||||
.as_ref()
|
||||
.expect("local slot should remain active"),
|
||||
&fixture.receiver
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_quarantines_still_unformatted_receiver_before_ready() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("unformatted minority should be isolated without changing quorum");
|
||||
assert!(fixture.ctx.local_disk_map().read().await[&fixture.receiver.endpoint().to_string()].is_none());
|
||||
assert!(fixture.ctx.local_disk_set_drives().read().await[0][0][3].is_none());
|
||||
assert!(fixture.set.disks.read().await[3].is_none());
|
||||
assert_eq!(fixture.set.disks.read().await.iter().flatten().count(), 3);
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("a later init retry must preserve completed quarantine");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_quarantines_foreign_completed_receiver_before_ready() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][3];
|
||||
write_format_file(&fixture.receiver, &foreign)
|
||||
.await
|
||||
.expect("foreign format should publish");
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("foreign minority should be isolated");
|
||||
assert!(fixture.ctx.local_disk_map().read().await[&fixture.receiver.endpoint().to_string()].is_none());
|
||||
assert!(fixture.ctx.local_disk_set_drives().read().await[0][0][3].is_none());
|
||||
assert!(fixture.set.disks.read().await[3].is_none());
|
||||
assert!(
|
||||
!fixture
|
||||
.ctx
|
||||
.local_disk_id_map()
|
||||
.read()
|
||||
.await
|
||||
.contains_key(&foreign.erasure.this)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_rechecks_replaced_receiver_after_io_and_accepts_current_member_on_retry() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
publish_pending_format(&fixture).await;
|
||||
let observed = load_format_erasure(&fixture.receiver, false)
|
||||
.await
|
||||
.expect("old receiver observation should load");
|
||||
let replacement = new_disk(
|
||||
&fixture.receiver.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement receiver should open");
|
||||
fixture
|
||||
.ctx
|
||||
.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(replacement.endpoint().to_string(), Some(replacement.clone()));
|
||||
fixture.ctx.local_disk_set_drives().write().await[0][0][3] = Some(replacement.clone());
|
||||
*fixture.set.disks.write().await.get_mut(3).expect("last slot should exist") = Some(replacement.clone());
|
||||
assert!(
|
||||
fixture
|
||||
.pending
|
||||
.publish_observation(&fixture.ctx, &fixture.set, None, Some(&fixture.receiver), Ok(observed))
|
||||
.await
|
||||
.is_err(),
|
||||
"stale I/O must not publish over a replacement"
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.ctx.local_disk_map().read().await[&replacement.endpoint().to_string()]
|
||||
.as_ref()
|
||||
.expect("replacement must not be quarantined"),
|
||||
&replacement
|
||||
));
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("existing init retry should validate the current replacement");
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.set.disks.read().await[3]
|
||||
.as_ref()
|
||||
.expect("validated slot should exist"),
|
||||
&replacement
|
||||
));
|
||||
assert_eq!(
|
||||
replacement.get_disk_id().await.expect("replacement ID should read"),
|
||||
Some(fixture.format.erasure.sets[0][3])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_remote_pending_publication_checks_shared_identity_and_slot() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let format = FormatV3::new(1, 4);
|
||||
let mut endpoint = Endpoint::try_from("http://remote-server:9000/data").expect("remote endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(3);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remote client should initialize without connecting");
|
||||
let mut endpoints = vec![endpoint.clone(); 4];
|
||||
for (index, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_disk_index(index);
|
||||
}
|
||||
let set = crate::set_disk::SetDisks::new_with_instance_ctx(
|
||||
"remote-pending-test".to_string(),
|
||||
Arc::new(tokio::sync::RwLock::new(vec![None; 4])),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
format.clone(),
|
||||
Vec::new(),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
let pending = PendingFormatDisk { disk: disk.clone() };
|
||||
let mut wrong_slot = format.clone();
|
||||
wrong_slot.erasure.this = format.erasure.sets[0][2];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(wrong_slot))
|
||||
.await
|
||||
.expect("wrong-slot format should remain excluded");
|
||||
assert!(set.disks.read().await[3].is_none());
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][3];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(foreign))
|
||||
.await
|
||||
.expect("foreign remote format should remain excluded");
|
||||
assert!(set.disks.read().await[3].is_none());
|
||||
let mut matching = format.clone();
|
||||
matching.erasure.this = format.erasure.sets[0][3];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(matching))
|
||||
.await
|
||||
.expect("exact remote slot should restore");
|
||||
assert!(Arc::ptr_eq(set.disks.read().await[3].as_ref().expect("remote slot should exist"), &disk));
|
||||
assert_eq!(
|
||||
disk.get_disk_id().await.expect("remote ID state should read"),
|
||||
Some(format.erasure.sets[0][3])
|
||||
);
|
||||
assert!(
|
||||
ctx.local_disk_map().read().await.is_empty(),
|
||||
"remote restoration must not register a local receiver"
|
||||
);
|
||||
}
|
||||
|
||||
async fn two_local_disks_with_missing_third() -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
|
||||
let (temp_dir, mut disks) = local_disks(2).await;
|
||||
disks.push(None);
|
||||
|
||||
@@ -431,8 +431,8 @@ pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
pub use bucket_fence::BucketIncarnationFenceGuard;
|
||||
pub(crate) mod delete_recovery_runtime;
|
||||
mod heal;
|
||||
pub(crate) use heal::{BucketHealScope, bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
mod init;
|
||||
@@ -494,6 +494,7 @@ pub struct ECStore {
|
||||
/// Memoizes bucket-incarnation validation under continuous lifecycle
|
||||
/// read-lock coverage (see [`bucket_fence`]).
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
pub(crate) ordinary_classification: crate::bucket::artifact::OrdinaryClassificationRegistry,
|
||||
}
|
||||
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
|
||||
@@ -1430,7 +1431,7 @@ impl BucketOperations for ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||
Box::pin(self.handle_make_bucket(bucket, opts)).await
|
||||
self.handle_make_bucket(bucket, opts).await
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -1904,6 +1905,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
};
|
||||
|
||||
let rendered = format!("{store:?}");
|
||||
@@ -1919,6 +1921,19 @@ mod tests {
|
||||
assert!(!rendered.contains("DiskHealthTracker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_bucket_handler_future_remains_stack_bounded() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let opts = MakeBucketOptions::default();
|
||||
let future = store.handle_make_bucket("stack-bounded-bucket", &opts);
|
||||
let future_size = std::mem::size_of_val(&future);
|
||||
|
||||
assert!(
|
||||
future_size <= 4 * 1024,
|
||||
"make bucket handler future must remain stack-bounded; measured {future_size} bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_options_debug_does_not_expand_tier_store_handle() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
@@ -1979,6 +1994,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -602,7 +602,7 @@ impl ECStore {
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||
check_new_object_args(bucket, object, opts.data_movement)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
|
||||
if self.single_pool() {
|
||||
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
|
||||
@@ -741,10 +741,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -778,7 +775,7 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart upload requires data_movement options"));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
let pool = self.pools.get(target_pool_idx).ok_or_else(|| {
|
||||
Error::InvalidArgument("data-movement".to_string(), "target-pool".to_string(), target_pool_idx.to_string())
|
||||
@@ -1160,7 +1157,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -1261,7 +1258,7 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
|
||||
let expected_incarnation_id = opts
|
||||
@@ -1666,6 +1663,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+600
-453
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,313 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Durable restrictions owned by an authenticated user. A record is not an IAM
|
||||
//! grant or credential; issuing credentials still requires current IAM checks.
|
||||
use crate::{error::is_err_config_not_found, store::Store};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const PREFIX: &str = "config/iam/plugin-delegations/v1";
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Restrictions {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
|
||||
pub version: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
|
||||
pub output_bucket: Option<String>,
|
||||
pub deployment: String,
|
||||
pub bucket: String,
|
||||
pub input_prefix: String,
|
||||
pub output_prefix: String,
|
||||
pub expires_at_unix_seconds: u64,
|
||||
}
|
||||
|
||||
// An absent v1 field is compatible; an explicitly null v2 field is invalid.
|
||||
fn present_value<'de, T: Deserialize<'de>, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Option<T>, D::Error> {
|
||||
T::deserialize(deserializer).map(Some)
|
||||
}
|
||||
|
||||
impl Restrictions {
|
||||
pub fn output_bucket(&self) -> Result<&str, DelegationError> {
|
||||
match (self.version, self.output_bucket.as_deref()) {
|
||||
(None, None) => Ok(&self.bucket),
|
||||
(Some(2), Some(output)) => Ok(output),
|
||||
_ => Err(DelegationError::Invalid),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self, now: u64) -> Result<(), DelegationError> {
|
||||
let output_bucket = self.output_bucket()?;
|
||||
let names: [&str; 5] = [
|
||||
&self.deployment,
|
||||
&self.bucket,
|
||||
&self.input_prefix,
|
||||
&self.output_prefix,
|
||||
output_bucket,
|
||||
];
|
||||
if names
|
||||
.iter()
|
||||
.any(|s| s.is_empty() || s.len() > 1024 || s.contains("${") || s.chars().any(char::is_control))
|
||||
|| self.bucket.contains('/')
|
||||
|| self.bucket.contains('*')
|
||||
|| self.bucket.contains('?')
|
||||
|| output_bucket.contains('/')
|
||||
|| output_bucket.contains('*')
|
||||
|| output_bucket.contains('?')
|
||||
|| self.input_prefix.contains('*')
|
||||
|| self.input_prefix.contains('?')
|
||||
|| self.output_prefix.contains('*')
|
||||
|| self.output_prefix.contains('?')
|
||||
|| self.expires_at_unix_seconds <= now
|
||||
|| self.expires_at_unix_seconds.saturating_sub(now) > 86400
|
||||
{
|
||||
return Err(DelegationError::Invalid);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Delegation {
|
||||
pub version: u32,
|
||||
pub id: String,
|
||||
pub revision: String,
|
||||
pub owner_access_key: String,
|
||||
pub created_at_unix_seconds: u64,
|
||||
pub restrictions: Restrictions,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Tombstone {
|
||||
version: u32,
|
||||
id: String,
|
||||
revision: String,
|
||||
owner_access_key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum DelegationError {
|
||||
#[error("invalid delegation restriction")]
|
||||
Invalid,
|
||||
#[error("delegation not found")]
|
||||
NotFound,
|
||||
#[error("delegation owner rejected")]
|
||||
OwnerRejected,
|
||||
#[error("delegation unavailable")]
|
||||
Unavailable,
|
||||
#[error("delegation revoked")]
|
||||
Revoked,
|
||||
#[error("delegation expired")]
|
||||
Expired,
|
||||
}
|
||||
|
||||
fn record_path(id: &str) -> Result<String, DelegationError> {
|
||||
if id.len() != 32 || !id.bytes().all(|c| c.is_ascii_hexdigit()) {
|
||||
return Err(DelegationError::Invalid);
|
||||
}
|
||||
Ok(format!("{PREFIX}/{id}/record.json"))
|
||||
}
|
||||
|
||||
async fn load<T: Store>(store: &T, id: &str) -> Result<Delegation, DelegationError> {
|
||||
let record: Delegation = store.load_iam_config_strict(record_path(id)?).await.map_err(|error| {
|
||||
if is_err_config_not_found(&error) {
|
||||
DelegationError::NotFound
|
||||
} else {
|
||||
DelegationError::Unavailable
|
||||
}
|
||||
})?;
|
||||
if !matches!(record.version, 1 | 2)
|
||||
|| record.version != record.restrictions.version.unwrap_or(1)
|
||||
|| record.id != id
|
||||
|| record.revision != id
|
||||
|| record.owner_access_key.is_empty()
|
||||
{
|
||||
return Err(DelegationError::Unavailable);
|
||||
}
|
||||
record
|
||||
.restrictions
|
||||
.validate(record.created_at_unix_seconds)
|
||||
.map_err(|_| DelegationError::Unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
pub(crate) async fn create<T: Store>(
|
||||
store: &T,
|
||||
owner: &str,
|
||||
restrictions: Restrictions,
|
||||
now: u64,
|
||||
) -> Result<Delegation, DelegationError> {
|
||||
restrictions.validate(now)?;
|
||||
if owner.is_empty() {
|
||||
return Err(DelegationError::OwnerRejected);
|
||||
}
|
||||
// IDs are generated exclusively here; no update operation accepts an ID.
|
||||
let id = format!("{:032x}", rand::random::<u128>());
|
||||
let path = record_path(&id)?;
|
||||
match store.load_iam_config_strict::<Delegation>(&path).await {
|
||||
Err(error) if is_err_config_not_found(&error) => {}
|
||||
_ => return Err(DelegationError::Unavailable),
|
||||
}
|
||||
let record = Delegation {
|
||||
version: restrictions.version.unwrap_or(1),
|
||||
revision: id.clone(),
|
||||
id,
|
||||
owner_access_key: owner.to_owned(),
|
||||
created_at_unix_seconds: now,
|
||||
restrictions,
|
||||
};
|
||||
store
|
||||
.save_iam_config(&record, path)
|
||||
.await
|
||||
.map_err(|_| DelegationError::Unavailable)?;
|
||||
if load(store, &record.id).await? != record {
|
||||
return Err(DelegationError::Unavailable);
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
pub(crate) async fn revoke<T: Store>(store: &T, owner: &str, id: &str) -> Result<(), DelegationError> {
|
||||
let record = load(store, id).await?;
|
||||
if record.owner_access_key != owner {
|
||||
return Err(DelegationError::OwnerRejected);
|
||||
}
|
||||
let tombstone = Tombstone {
|
||||
version: 1,
|
||||
id: record.id,
|
||||
revision: record.revision,
|
||||
owner_access_key: record.owner_access_key,
|
||||
};
|
||||
let path = format!("{PREFIX}/{id}/revoked.json");
|
||||
// Every retry writes exactly the same revocation; active records never change.
|
||||
store
|
||||
.save_iam_config(&tombstone, &path)
|
||||
.await
|
||||
.map_err(|_| DelegationError::Unavailable)?;
|
||||
let saved: Tombstone = store
|
||||
.load_iam_config_strict(path)
|
||||
.await
|
||||
.map_err(|_| DelegationError::Unavailable)?;
|
||||
if saved != tombstone {
|
||||
return Err(DelegationError::Unavailable);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn read_active<T: Store>(store: &T, owner: &str, id: &str, now: u64) -> Result<Delegation, DelegationError> {
|
||||
let record = load(store, id).await?;
|
||||
if record.owner_access_key != owner {
|
||||
return Err(DelegationError::OwnerRejected);
|
||||
}
|
||||
check_active(store, record, now).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_owned<T: Store>(store: &T, owner: &str, id: &str) -> Result<Delegation, DelegationError> {
|
||||
let record = load(store, id).await?;
|
||||
if record.owner_access_key != owner {
|
||||
return Err(DelegationError::OwnerRejected);
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_active_for_broker<T: Store>(store: &T, id: &str, now: u64) -> Result<Delegation, DelegationError> {
|
||||
let record = load(store, id).await?;
|
||||
check_active(store, record, now).await
|
||||
}
|
||||
async fn check_active<T: Store>(store: &T, record: Delegation, now: u64) -> Result<Delegation, DelegationError> {
|
||||
let id = &record.id;
|
||||
match store
|
||||
.load_iam_config_strict::<Tombstone>(format!("{PREFIX}/{id}/revoked.json"))
|
||||
.await
|
||||
{
|
||||
Ok(_) => return Err(DelegationError::Revoked),
|
||||
Err(error) if is_err_config_not_found(&error) => {}
|
||||
Err(_) => return Err(DelegationError::Unavailable),
|
||||
}
|
||||
if now >= record.restrictions.expires_at_unix_seconds {
|
||||
return Err(DelegationError::Expired);
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn delegation_v1_json_remains_compatible() {
|
||||
let legacy = serde_json::json!({"deployment":"runner-a","bucket":"photos",
|
||||
"input_prefix":"inputs/","output_prefix":"outputs/","expires_at_unix_seconds":200});
|
||||
let restriction: Restrictions = serde_json::from_value(legacy.clone()).expect("legacy restriction");
|
||||
restriction.validate(100).expect("legacy validation");
|
||||
assert_eq!(restriction.output_bucket().expect("v1 output"), "photos");
|
||||
assert_eq!(serde_json::to_value(restriction).expect("legacy serialization"), legacy);
|
||||
let record = serde_json::json!({"version":1,"id":"a".repeat(32),"revision":"a".repeat(32),
|
||||
"owner_access_key":"owner","created_at_unix_seconds":100,"restrictions":legacy});
|
||||
let decoded: Delegation = serde_json::from_value(record.clone()).expect("legacy record");
|
||||
assert_eq!(serde_json::to_value(decoded).expect("legacy record serialization"), record);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delegation_v2_requires_explicit_output_bucket() {
|
||||
let valid = serde_json::json!({"version":2,"deployment":"runner-a","bucket":"photos",
|
||||
"input_prefix":"inputs/","output_bucket":"artifacts","output_prefix":"outputs/",
|
||||
"expires_at_unix_seconds":200});
|
||||
let restriction: Restrictions = serde_json::from_value(valid.clone()).expect("v2 restriction");
|
||||
restriction.validate(100).expect("explicit output bucket");
|
||||
assert_eq!(restriction.output_bucket().expect("v2 output"), "artifacts");
|
||||
assert_eq!(serde_json::to_value(restriction).expect("v2 serialization"), valid);
|
||||
for key in ["version", "output_bucket"] {
|
||||
let mut bad = valid.clone();
|
||||
bad.as_object_mut().expect("object").remove(key);
|
||||
assert!(
|
||||
serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
|
||||
"v2 required {key}"
|
||||
);
|
||||
}
|
||||
for (key, value) in [
|
||||
("version", serde_json::json!(1)),
|
||||
("version", serde_json::json!(3)),
|
||||
("version", serde_json::Value::Null),
|
||||
("output_bucket", serde_json::Value::Null),
|
||||
("output_bucket", serde_json::json!("")),
|
||||
("output_bucket", serde_json::json!("other/*")),
|
||||
("output_bucket", serde_json::json!("${aws:username}")),
|
||||
("output_bucket", serde_json::json!("other?")),
|
||||
("output_bucket", serde_json::json!("other\n")),
|
||||
("unknown", serde_json::json!(true)),
|
||||
] {
|
||||
let mut bad = valid.clone();
|
||||
bad[key] = value;
|
||||
assert!(
|
||||
serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
|
||||
"invalid v2 {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restrictions_are_literal_bounded_and_expiring() {
|
||||
let mut restriction = Restrictions {
|
||||
version: None,
|
||||
output_bucket: None,
|
||||
deployment: "runner-a".into(),
|
||||
bucket: "photos".into(),
|
||||
input_prefix: "inputs/".into(),
|
||||
output_prefix: "outputs/".into(),
|
||||
expires_at_unix_seconds: 200,
|
||||
};
|
||||
assert!(restriction.validate(100).is_ok());
|
||||
restriction.output_prefix = "outputs/*".into();
|
||||
assert!(matches!(restriction.validate(100), Err(DelegationError::Invalid)));
|
||||
restriction.output_prefix = "outputs/".into();
|
||||
assert!(matches!(restriction.validate(200), Err(DelegationError::Invalid)));
|
||||
restriction.expires_at_unix_seconds = 200;
|
||||
restriction.input_prefix = "inputs/${aws:username}/".into();
|
||||
assert!(
|
||||
matches!(restriction.validate(100), Err(DelegationError::Invalid)),
|
||||
"C07_LITERAL_SCOPE_BOUND: policy variables are not literal restrictions"
|
||||
);
|
||||
assert!(record_path("../../users").is_err());
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ pub(crate) const EVENT_OIDC_DIAGNOSTICS: &str = "oidc_diagnostics";
|
||||
const EVENT_OIDC_STATE: &str = "oidc_state";
|
||||
|
||||
pub mod cache;
|
||||
pub mod delegation;
|
||||
pub mod error;
|
||||
pub mod federation;
|
||||
pub mod keyring;
|
||||
|
||||
+30
-42
@@ -3409,27 +3409,23 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_jwt_claims_basic() {
|
||||
let user_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: Credentials {
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
session_token: "invalid-token".to_string(), // Invalid token for testing error handling
|
||||
expiration: None,
|
||||
status: "enabled".to_string(),
|
||||
parent_user: "".to_string(),
|
||||
groups: None,
|
||||
claims: Some({
|
||||
let mut claims = HashMap::new();
|
||||
claims.insert("sub".to_string(), json!("test-user"));
|
||||
claims.insert("aud".to_string(), json!("test-audience"));
|
||||
claims
|
||||
}),
|
||||
name: None,
|
||||
description: None,
|
||||
},
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let user_identity = UserIdentity::new(Credentials {
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
session_token: "invalid-token".to_string(), // Invalid token for testing error handling
|
||||
expiration: None,
|
||||
status: "enabled".to_string(),
|
||||
parent_user: "".to_string(),
|
||||
groups: None,
|
||||
claims: Some({
|
||||
let mut claims = HashMap::new();
|
||||
claims.insert("sub".to_string(), json!("test-user"));
|
||||
claims.insert("aud".to_string(), json!("test-audience"));
|
||||
claims
|
||||
}),
|
||||
name: None,
|
||||
description: None,
|
||||
});
|
||||
|
||||
let result = extract_jwt_claims(&user_identity);
|
||||
// In test environment without proper JWT setup, this should fail
|
||||
@@ -3438,22 +3434,18 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_jwt_claims_no_claims() {
|
||||
let user_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: Credentials {
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
session_token: "".to_string(), // Empty token
|
||||
expiration: None,
|
||||
status: "enabled".to_string(),
|
||||
parent_user: "".to_string(),
|
||||
groups: None,
|
||||
claims: None,
|
||||
name: None,
|
||||
description: None,
|
||||
},
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let user_identity = UserIdentity::new(Credentials {
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
session_token: "".to_string(), // Empty token
|
||||
expiration: None,
|
||||
status: "enabled".to_string(),
|
||||
parent_user: "".to_string(),
|
||||
groups: None,
|
||||
claims: None,
|
||||
name: None,
|
||||
description: None,
|
||||
});
|
||||
|
||||
let result = extract_jwt_claims(&user_identity);
|
||||
// Should fail with empty session token
|
||||
@@ -3512,11 +3504,7 @@ mod tests {
|
||||
description: None,
|
||||
};
|
||||
|
||||
let user_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials,
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let user_identity = UserIdentity::new(credentials);
|
||||
|
||||
// Test basic structure
|
||||
assert_eq!(user_identity.version, 1);
|
||||
|
||||
@@ -91,6 +91,7 @@ pub(crate) mod crate_boundary {
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
|
||||
}
|
||||
|
||||
pub(crate) mod runtime {
|
||||
|
||||
@@ -26,6 +26,11 @@ pub trait Store: Clone + Send + Sync + 'static {
|
||||
fn has_watcher(&self) -> bool;
|
||||
async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()>;
|
||||
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item>;
|
||||
/// Only confirmed storage absence may return ConfigNotFound. Corrupt or
|
||||
/// undecryptable objects must remain errors, especially for revocations.
|
||||
async fn load_iam_config_strict<Item: DeserializeOwned>(&self, _path: impl AsRef<str> + Send) -> Result<Item> {
|
||||
Err(crate::error::Error::IamSysNotInitialized)
|
||||
}
|
||||
async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
|
||||
|
||||
async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;
|
||||
|
||||
@@ -42,6 +42,7 @@ use tracing::{debug, error, warn};
|
||||
|
||||
use crate::storage_api::object_store::{
|
||||
HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
read_config_preserve_empty_with_metadata_opts,
|
||||
};
|
||||
|
||||
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
|
||||
@@ -414,9 +415,14 @@ impl ObjectStore {
|
||||
&self,
|
||||
path: impl AsRef<str> + Send,
|
||||
mode: LoadMode,
|
||||
preserve_empty: bool,
|
||||
) -> Result<(Vec<u8>, IamObjectInfo)> {
|
||||
let path_ref = path.as_ref();
|
||||
let (data, obj) = read_iam_config_with_metadata(self.object_api.clone(), path_ref, &mode.read_opts()).await?;
|
||||
let (data, obj) = if preserve_empty {
|
||||
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path_ref, &mode.read_opts()).await?
|
||||
} else {
|
||||
read_iam_config_with_metadata(self.object_api.clone(), path_ref, &mode.read_opts()).await?
|
||||
};
|
||||
let outcome = Self::decrypt_data_with_source(&data)?;
|
||||
self.maybe_schedule_lazy_rewrite(path_ref, &outcome, &obj);
|
||||
|
||||
@@ -611,11 +617,17 @@ impl ObjectStore {
|
||||
Ok(serde_json::from_slice(&outcome.plain)?)
|
||||
}
|
||||
|
||||
fn decode_iam_config_strict<Item: DeserializeOwned>(data: &[u8]) -> Result<Item> {
|
||||
let outcome = Self::decrypt_data_with_source(data)?;
|
||||
Ok(serde_json::from_slice(&outcome.plain)?)
|
||||
}
|
||||
|
||||
/// Parameterized core of [`Store::load_user_identity`].
|
||||
async fn load_user_identity_with(&self, name: &str, user_type: UserType, mode: LoadMode) -> Result<UserIdentity> {
|
||||
let mut u: UserIdentity = self
|
||||
.load_iam_config_with(get_user_identity_path(name, user_type), mode)
|
||||
.load_iamconfig_bytes_with_metadata(get_user_identity_path(name, user_type), mode, true)
|
||||
.await
|
||||
.and_then(|(data, _)| serde_json::from_slice(&data).map_err(Error::from))
|
||||
.map_err(|err| {
|
||||
if is_err_config_not_found(&err) {
|
||||
debug!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity missing");
|
||||
@@ -695,7 +707,7 @@ impl ObjectStore {
|
||||
/// Parameterized core of [`Store::load_policy`].
|
||||
async fn load_policy_with(&self, name: &str, mode: LoadMode) -> Result<PolicyDoc> {
|
||||
let (data, obj) = self
|
||||
.load_iamconfig_bytes_with_metadata(get_policy_doc_path(name), mode)
|
||||
.load_iamconfig_bytes_with_metadata(get_policy_doc_path(name), mode, false)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if is_err_config_not_found(&err) {
|
||||
@@ -824,6 +836,13 @@ impl Store for ObjectStore {
|
||||
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
|
||||
self.load_iam_config_with(path, LoadMode::Locked).await
|
||||
}
|
||||
async fn load_iam_config_strict<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
|
||||
let (data, _) =
|
||||
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path.as_ref(), &LoadMode::Locked.read_opts())
|
||||
.await?;
|
||||
Self::decode_iam_config_strict(&data)
|
||||
}
|
||||
|
||||
/// Saves IAM configuration with a retry mechanism on failure.
|
||||
///
|
||||
/// Attempts to save the IAM configuration up to 5 times if the storage layer is not ready,
|
||||
@@ -1458,6 +1477,40 @@ mod tests {
|
||||
assert!(result.is_err(), "corrupt stream_io data should fail decrypt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_revocation_load_keeps_real_crypto_failure_distinct_from_absence() {
|
||||
let cred = test_cred();
|
||||
let key = format!("{}:{}", cred.access_key, cred.secret_key);
|
||||
let plain = br#"{"version":1,"id":"revoked"}"#;
|
||||
let mut encrypted = rustfs_crypto::encrypt_stream_io(key.as_bytes(), plain).expect("real stored ciphertext");
|
||||
// Exercise the exact decoder used after the actual object read.
|
||||
assert!(ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted).is_ok());
|
||||
encrypted[50] ^= 0xFF;
|
||||
let error = ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted)
|
||||
.expect_err("corrupt tombstone must not become missing");
|
||||
assert!(
|
||||
matches!(error, Error::CryptoError(_)),
|
||||
"C07_REVOCATION_FAIL_CLOSED: preserve actual crypto error"
|
||||
);
|
||||
assert!(
|
||||
!crate::error::is_err_config_not_found(&error),
|
||||
"C07_REVOCATION_FAIL_CLOSED: corrupted revocation is not absence"
|
||||
);
|
||||
// The preserve-empty reader keeps an existing zero-byte object as
|
||||
// Ok(empty); ecstore's read_config_preserve_empty_distinguishes_empty_object
|
||||
// covers that storage boundary. Its payload must fail here, not become
|
||||
// ConfigNotFound and reactivate a revoked delegation.
|
||||
let empty =
|
||||
ObjectStore::decode_iam_config_strict::<serde_json::Value>(&[]).expect_err("an existing empty tombstone is corrupt");
|
||||
assert!(
|
||||
!crate::error::is_err_config_not_found(&empty),
|
||||
"C07_REVOCATION_FAIL_CLOSED: empty existing revocation is not absence"
|
||||
);
|
||||
let invalid_json =
|
||||
ObjectStore::decode_iam_config_strict::<serde_json::Value>(br#"{"version": }"#).expect_err("invalid stored JSON");
|
||||
assert!(!crate::error::is_err_config_not_found(&invalid_json));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decrypt_data_short_data_fails() {
|
||||
let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON
|
||||
|
||||
+265
-27
@@ -75,6 +75,7 @@ pub const STATUS_ENABLED: &str = "enabled";
|
||||
pub const STATUS_DISABLED: &str = "disabled";
|
||||
|
||||
pub const POLICYNAME: &str = "policy";
|
||||
pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
|
||||
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
|
||||
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
|
||||
const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-policy";
|
||||
@@ -168,6 +169,12 @@ enum PreparedServicePolicyMode {
|
||||
|
||||
#[derive(Clone)]
|
||||
enum PreparedIamMode {
|
||||
TerminalDeny,
|
||||
PluginAttemptSts {
|
||||
is_owner: bool,
|
||||
combined_policy: Policy,
|
||||
session_policy: PreparedSessionPolicy,
|
||||
},
|
||||
Opa,
|
||||
Owner,
|
||||
Deny,
|
||||
@@ -202,7 +209,7 @@ impl PreparedIamAuth {
|
||||
pub async fn needs_existing_object_tag_for_args(&self, args: &Args<'_>) -> bool {
|
||||
match &self.mode {
|
||||
PreparedIamMode::Opa => true,
|
||||
PreparedIamMode::Owner | PreparedIamMode::Deny => false,
|
||||
PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => false,
|
||||
PreparedIamMode::Regular { combined_policy } => {
|
||||
policy_needs_existing_object_tag_for_args(combined_policy, args).await
|
||||
}
|
||||
@@ -210,6 +217,11 @@ impl PreparedIamAuth {
|
||||
combined_policy,
|
||||
session_policy,
|
||||
..
|
||||
}
|
||||
| PreparedIamMode::PluginAttemptSts {
|
||||
combined_policy,
|
||||
session_policy,
|
||||
..
|
||||
} => {
|
||||
policy_needs_existing_object_tag_for_args(combined_policy, args).await
|
||||
|| prepared_session_policy_needs_existing_object_tag_for_args(session_policy, args).await
|
||||
@@ -239,14 +251,56 @@ impl PreparedIamAuth {
|
||||
pub fn combined_policy_for_view(&self) -> Option<&Policy> {
|
||||
match &self.mode {
|
||||
PreparedIamMode::Regular { combined_policy } => Some(combined_policy),
|
||||
PreparedIamMode::Sts { combined_policy, .. } => Some(combined_policy),
|
||||
PreparedIamMode::Sts { combined_policy, .. } | PreparedIamMode::PluginAttemptSts { combined_policy, .. } => {
|
||||
Some(combined_policy)
|
||||
}
|
||||
PreparedIamMode::ServiceAccount { combined_policy, .. } => Some(combined_policy),
|
||||
PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny => None,
|
||||
PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Store> IamSys<T> {
|
||||
/// For the authenticated, explicitly configured broker handler only.
|
||||
pub async fn read_active_plugin_delegation_for_broker(
|
||||
&self,
|
||||
id: &str,
|
||||
now: u64,
|
||||
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
|
||||
crate::delegation::read_active_for_broker(&self.store.api, id, now).await
|
||||
}
|
||||
pub async fn create_plugin_delegation(
|
||||
&self,
|
||||
owner: &str,
|
||||
restrictions: crate::delegation::Restrictions,
|
||||
now: u64,
|
||||
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
|
||||
crate::delegation::create(&self.store.api, owner, restrictions, now).await
|
||||
}
|
||||
pub async fn revoke_plugin_delegation(
|
||||
&self,
|
||||
owner: &str,
|
||||
id: &str,
|
||||
) -> std::result::Result<(), crate::delegation::DelegationError> {
|
||||
crate::delegation::revoke(&self.store.api, owner, id).await
|
||||
}
|
||||
pub async fn read_active_plugin_delegation(
|
||||
&self,
|
||||
owner: &str,
|
||||
id: &str,
|
||||
now: u64,
|
||||
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
|
||||
crate::delegation::read_active(&self.store.api, owner, id, now).await
|
||||
}
|
||||
/// Reads an owned delegation record for historical lookups, including expired or revoked records.
|
||||
pub async fn read_owned_plugin_delegation(
|
||||
&self,
|
||||
owner: &str,
|
||||
id: &str,
|
||||
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
|
||||
crate::delegation::read_owned(&self.store.api, owner, id).await
|
||||
}
|
||||
|
||||
/// Create a new IamSys instance with the given IamCache store
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -281,6 +335,16 @@ impl<T: Store> IamSys<T> {
|
||||
*guard = PolicyPluginState::Ready(client);
|
||||
}
|
||||
|
||||
/// Restricted credentials require the built-in parent/session intersection.
|
||||
/// External authorization bypasses that path, so only Disabled is supported.
|
||||
pub async fn ensure_restricted_credential_vending() -> Result<()> {
|
||||
if matches!(Self::policy_plugin_state().await, PolicyPluginState::Disabled) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::IAMActionNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
|
||||
let policy_plugin_state = get_policy_plugin_state();
|
||||
let guard = policy_plugin_state.read().await;
|
||||
@@ -1383,17 +1447,50 @@ impl<T: Store> IamSys<T> {
|
||||
/// 1) know whether policy evaluation may need `s3:ExistingObjectTag`, and
|
||||
/// 2) evaluate with final conditions without re-merging identity policies.
|
||||
pub async fn prepare_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
|
||||
if args.is_owner {
|
||||
let marked = args.claims.contains_key(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
|
||||
if args.is_owner && !marked {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Owner,
|
||||
};
|
||||
}
|
||||
|
||||
if let Some(prepared) = Self::policy_plugin_state().await.prepared_iam_auth() {
|
||||
// One observed state decides this request's path. Marked credentials
|
||||
// never enter OPA, including after a restart or on another node.
|
||||
let state = Self::policy_plugin_state().await;
|
||||
if marked {
|
||||
if args.is_owner
|
||||
|| args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
|
||||
|| !matches!(state, PolicyPluginState::Disabled)
|
||||
{
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
}
|
||||
let prepared = self.prepare_builtin_auth(args).await;
|
||||
let mode = match prepared.mode {
|
||||
PreparedIamMode::Sts {
|
||||
is_owner,
|
||||
combined_policy,
|
||||
session_policy,
|
||||
} => PreparedIamMode::PluginAttemptSts {
|
||||
is_owner,
|
||||
combined_policy,
|
||||
session_policy,
|
||||
},
|
||||
_ => PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: prepared.needs_existing_object_tag,
|
||||
mode,
|
||||
};
|
||||
}
|
||||
if let Some(prepared) = state.prepared_iam_auth() {
|
||||
return prepared;
|
||||
}
|
||||
|
||||
self.prepare_builtin_auth(args).await
|
||||
}
|
||||
async fn prepare_builtin_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
|
||||
let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
@@ -1425,6 +1522,20 @@ impl<T: Store> IamSys<T> {
|
||||
/// bucket policies on an IAM denial. An error must not become an implicit deny.
|
||||
pub async fn try_eval_prepared(&self, prepared: &PreparedIamAuth, args: &Args<'_>) -> Result<bool> {
|
||||
Ok(match &prepared.mode {
|
||||
PreparedIamMode::TerminalDeny => return Err(Error::IAMActionNotAllowed),
|
||||
PreparedIamMode::PluginAttemptSts {
|
||||
is_owner,
|
||||
combined_policy,
|
||||
session_policy,
|
||||
} => {
|
||||
// A false IAM result permits bucket-policy fallback. Restricted
|
||||
// plugin credentials must instead terminate authorization.
|
||||
let session_allowed = evaluate_prepared_session_policy(session_policy, args).await == Some(true);
|
||||
if !session_allowed || !(*is_owner || combined_policy.is_allowed(args).await) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
true
|
||||
}
|
||||
PreparedIamMode::Opa => {
|
||||
let Some(opa_enable) = Self::get_policy_plugin_client().await else {
|
||||
tracing::warn!("eval_prepared: OPA mode requested but plugin is unavailable");
|
||||
@@ -2127,6 +2238,145 @@ mod tests {
|
||||
assert!(prepared.combined_policy_for_view().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn marked_signed_sts_denial_is_terminal_before_public_bucket_fallback() {
|
||||
let iam = IamSys::new(IamCache::new(StsTestMockStore::new(false)).await.unwrap());
|
||||
let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
|
||||
let signing_key = "c07-test-token-signing-key";
|
||||
let mut metadata=HashMap::from([
|
||||
("parent".into(),Value::String("sts-fallback-test-parent".into())),
|
||||
("exp".into(),serde_json::json!((OffsetDateTime::now_utc()+time::Duration::hours(1)).unix_timestamp())),
|
||||
(SESSION_POLICY_NAME.into(),Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/inside"]}]}"#))),
|
||||
]);
|
||||
metadata.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), Value::Bool(true));
|
||||
let mut marked = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
|
||||
marked.parent_user = "sts-fallback-test-parent".into();
|
||||
let marked_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&marked.session_token, signing_key)
|
||||
.expect("signed test credentials must decode")
|
||||
.claims;
|
||||
let marked_user = UserIdentity::from(marked.clone());
|
||||
iam.store
|
||||
.cache
|
||||
.add_or_update_sts_account(&marked.access_key, &marked_user, OffsetDateTime::now_utc());
|
||||
metadata.remove(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
|
||||
let mut ordinary = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
|
||||
ordinary.parent_user = marked.parent_user.clone();
|
||||
let ordinary_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&ordinary.session_token, signing_key)
|
||||
.expect("signed test credentials must decode")
|
||||
.claims;
|
||||
iam.store.cache.add_or_update_sts_account(
|
||||
&ordinary.access_key,
|
||||
&UserIdentity::from(ordinary.clone()),
|
||||
OffsetDateTime::now_utc(),
|
||||
);
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
let args = Args {
|
||||
account: &marked.access_key,
|
||||
groups: &groups,
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
bucket: "mybucket",
|
||||
object: "outside",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
claims: &marked_claims,
|
||||
deny_only: false,
|
||||
};
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
|
||||
let denied = iam.prepare_auth(&args).await;
|
||||
let disabled_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
let inside = Args {
|
||||
object: "inside",
|
||||
..args.clone()
|
||||
};
|
||||
let allowed = iam.prepare_auth(&inside).await;
|
||||
let inside_allowed = matches!(iam.try_eval_prepared(&allowed, &inside).await, Ok(true));
|
||||
let unmarked_args = Args {
|
||||
account: &ordinary.access_key,
|
||||
claims: &ordinary_claims,
|
||||
..args.clone()
|
||||
};
|
||||
let unmarked = iam.prepare_auth(&unmarked_args).await;
|
||||
let ordinary_false = matches!(iam.try_eval_prepared(&unmarked, &unmarked_args).await, Ok(false));
|
||||
let public:rustfs_policy::policy::BucketPolicy=serde_json::from_value(serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/*"]}]})).unwrap();
|
||||
let public_allows = public
|
||||
.is_allowed(&rustfs_policy::policy::BucketPolicyArgs {
|
||||
account: &marked.access_key,
|
||||
groups: &groups,
|
||||
action: args.action,
|
||||
bucket: args.bucket,
|
||||
object: args.object,
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
})
|
||||
.await;
|
||||
IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
|
||||
url: "http://127.0.0.1:8181/allow".into(),
|
||||
auth_token: String::new(),
|
||||
}))
|
||||
.await;
|
||||
let opa_marked = iam.prepare_auth(&args).await;
|
||||
let ready_terminal = matches!(iam.try_eval_prepared(&opa_marked, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
let opa_ordinary = iam.prepare_auth(&unmarked_args).await;
|
||||
let ordinary_keeps_opa = matches!(opa_ordinary.mode, PreparedIamMode::Opa);
|
||||
// The captured Disabled preparation stays on built-in intersection even
|
||||
// if an external policy client becomes ready before evaluation.
|
||||
let captured_still_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
|
||||
let pending = iam.prepare_auth(&args).await;
|
||||
let initializing_terminal = matches!(iam.try_eval_prepared(&pending, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Failed;
|
||||
let failed = iam.prepare_auth(&args).await;
|
||||
let failed_terminal = matches!(iam.try_eval_prepared(&failed, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
*get_policy_plugin_state().write().await = previous;
|
||||
assert!(public_allows, "C07_PUBLIC_BUCKET_CONTROL: real public policy permits outside object");
|
||||
assert!(
|
||||
disabled_terminal && captured_still_terminal,
|
||||
"C07_SCOPE_TERMINAL_BOUND: bucket policy must not rescue restricted scope denial"
|
||||
);
|
||||
assert!(
|
||||
ready_terminal && initializing_terminal && failed_terminal,
|
||||
"C07_EXTERNAL_POLICY_BOUND: marked persisted STS cannot enter OPA"
|
||||
);
|
||||
assert!(
|
||||
inside_allowed && ordinary_false && ordinary_keeps_opa,
|
||||
"C07_UNMARKED_COMPAT: preserve ordinary STS and valid marked scope"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn restricted_vending_rejects_actual_external_policy_plugin_state() {
|
||||
let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
|
||||
IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
|
||||
url: "http://127.0.0.1:8181/v1/data/rustfs/authz/allow".into(),
|
||||
auth_token: String::new(),
|
||||
}))
|
||||
.await;
|
||||
let active_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
|
||||
.await
|
||||
.is_err();
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
|
||||
let initializing_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
|
||||
.await
|
||||
.is_err();
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Failed;
|
||||
let failed_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
|
||||
.await
|
||||
.is_err();
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
|
||||
let disabled_allowed = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
|
||||
.await
|
||||
.is_ok();
|
||||
*get_policy_plugin_state().write().await = previous;
|
||||
assert!(
|
||||
active_rejected && initializing_rejected && failed_rejected,
|
||||
"C07_EXTERNAL_POLICY_BOUND: external authorization cannot bypass minted session scope"
|
||||
);
|
||||
assert!(disabled_allowed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_prepare_auth_requests_existing_object_tags_in_opa_mode() {
|
||||
@@ -2667,11 +2917,7 @@ mod tests {
|
||||
name: None,
|
||||
description: None,
|
||||
};
|
||||
let parent_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: creds,
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let parent_identity = UserIdentity::new(creds);
|
||||
let mut users = HashMap::new();
|
||||
users.insert(PARENT_USER.to_string(), parent_identity);
|
||||
|
||||
@@ -2703,11 +2949,7 @@ mod tests {
|
||||
name: None,
|
||||
description: None,
|
||||
};
|
||||
let parent_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: creds,
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let parent_identity = UserIdentity::new(creds);
|
||||
let mut users = HashMap::new();
|
||||
users.insert(PARENT_USER.to_string(), parent_identity);
|
||||
|
||||
@@ -2820,16 +3062,12 @@ mod tests {
|
||||
async fn group_writes_stamped_before_the_cache_load_time_still_publish() {
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let member = "group-stamp-member";
|
||||
let identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: Credentials {
|
||||
access_key: member.to_string(),
|
||||
secret_key: "longenoughsecret".to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let identity = UserIdentity::new(Credentials {
|
||||
access_key: member.to_string(),
|
||||
secret_key: "longenoughsecret".to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
iam_sys.store.cache.with_write_lock(|cache| {
|
||||
cache.add_or_update_user(member, &identity, OffsetDateTime::now_utc());
|
||||
// The startup load publishes every entity with the load time.
|
||||
|
||||
@@ -22,7 +22,11 @@
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::migration::try_migrate_iam_config;
|
||||
pub(crate) use rustfs_ecstore::api::config::com::save_config;
|
||||
pub(crate) use rustfs_ecstore::api::global::set_object_store_resolver;
|
||||
pub(crate) use rustfs_ecstore::api::layout::SetupType;
|
||||
pub(crate) use rustfs_ecstore::api::runtime::{InstanceContext, first_cluster_node_is_local, object_store_handle};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore;
|
||||
|
||||
// `update_erasure_type` is a write-side global facade entry. Its use is
|
||||
// restricted to reviewed storage_api boundaries; this test-only module is
|
||||
|
||||
@@ -33,12 +33,21 @@
|
||||
|
||||
mod ecstore_test_compat;
|
||||
|
||||
use ecstore_test_compat::fixture::{SetupType, update_erasure_type};
|
||||
use ecstore_test_compat::fixture::{
|
||||
ECStore, InstanceContext, SetupType, first_cluster_node_is_local, object_store_handle, save_config,
|
||||
set_object_store_resolver, update_erasure_type,
|
||||
};
|
||||
use rustfs_credentials::{Credentials, IAM_POLICY_CLAIM_NAME_SA};
|
||||
use rustfs_iam::cache::Cache;
|
||||
use rustfs_iam::store::object::ObjectStore;
|
||||
use rustfs_iam::store::{GroupInfo, Store};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::manager::{IamCache, IamState};
|
||||
use rustfs_iam::store::object::{IAM_CONFIG_POLICIES_PREFIX, IAM_CONFIG_USERS_PREFIX, ObjectStore};
|
||||
use rustfs_iam::store::{GroupInfo, Store, UserType};
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64};
|
||||
use std::sync::{Arc, Mutex, OnceLock, Weak};
|
||||
|
||||
const TEST_GROUP: &str = "seq-restart-group";
|
||||
const TEST_MEMBERS: [&str; 2] = ["alice", "bob"];
|
||||
@@ -69,7 +78,8 @@ async fn load_all_bypasses_namespace_lock_quorum() {
|
||||
.build()
|
||||
.await
|
||||
.ecstore;
|
||||
let store = ObjectStore::new(ecstore);
|
||||
let _route_guard = TestStoreRouteGuard::new(&ecstore);
|
||||
let store = ObjectStore::new(ecstore.clone());
|
||||
|
||||
// Seed IAM data while namespace locks still work (single-node mode).
|
||||
store
|
||||
@@ -128,3 +138,447 @@ async fn load_all_bypasses_namespace_lock_quorum() {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
const IDENTITY_PARENT: &str = "fixture-parent";
|
||||
const IDENTITY_CHILD: &str = "fixture-service-child";
|
||||
|
||||
static TEST_STORE_ROUTE: Mutex<Option<Weak<ECStore>>> = Mutex::new(None);
|
||||
static TEST_STORE_RESOLVER: OnceLock<()> = OnceLock::new();
|
||||
|
||||
struct TestStoreRouteGuard;
|
||||
|
||||
impl TestStoreRouteGuard {
|
||||
fn new(ecstore: &Arc<ECStore>) -> Self {
|
||||
TEST_STORE_RESOLVER.get_or_init(|| {
|
||||
assert!(
|
||||
set_object_store_resolver(Arc::new(|| {
|
||||
TEST_STORE_ROUTE
|
||||
.lock()
|
||||
.expect("lock test store route")
|
||||
.as_ref()
|
||||
.and_then(Weak::upgrade)
|
||||
})),
|
||||
"test binary must own its resolver registration"
|
||||
);
|
||||
});
|
||||
let mut route = TEST_STORE_ROUTE.lock().expect("lock test store route");
|
||||
assert!(route.is_none(), "another test store route is active");
|
||||
*route = Some(Arc::downgrade(ecstore));
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestStoreRouteGuard {
|
||||
fn drop(&mut self) {
|
||||
*TEST_STORE_ROUTE.lock().expect("lock test store route for cleanup") = None;
|
||||
}
|
||||
}
|
||||
|
||||
struct IdentityStoreFixture {
|
||||
previous_store: Option<Arc<ECStore>>,
|
||||
ecstore: Arc<ECStore>,
|
||||
manager: IamCache<ObjectStore>,
|
||||
_route_guard: TestStoreRouteGuard,
|
||||
_temp_dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl IdentityStoreFixture {
|
||||
async fn new() -> Self {
|
||||
let temp_dir = tempfile::TempDir::with_prefix("rustfs_iam_identity_fixture_").expect("create identity fixture directory");
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp_dir.path())
|
||||
.init_bucket_metadata(false)
|
||||
.instance_ctx(ctx.clone())
|
||||
.build()
|
||||
.await;
|
||||
ctx.set_endpoints(env.endpoint_pools.clone());
|
||||
let ecstore = env.ecstore;
|
||||
let previous_store = object_store_handle();
|
||||
let route_guard = TestStoreRouteGuard::new(&ecstore);
|
||||
let selected = object_store_handle().expect("fixture route selects its ECStore");
|
||||
assert!(Arc::ptr_eq(&selected, &ecstore), "fixture route must select the isolated ECStore");
|
||||
drop(selected);
|
||||
assert!(first_cluster_node_is_local().await, "isolated fixture has a local first endpoint");
|
||||
temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], rustfs_iam::build_iam_sys(ecstore.clone()))
|
||||
.await
|
||||
.expect("initialize IAM metadata through production owner");
|
||||
let store = ObjectStore::new(ecstore.clone());
|
||||
let parent = UserIdentity::new(Credentials {
|
||||
access_key: IDENTITY_PARENT.to_string(),
|
||||
secret_key: "fixture-parent-secret".to_string(),
|
||||
expiration: None,
|
||||
session_token: String::new(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
let child = UserIdentity::new(Credentials {
|
||||
access_key: IDENTITY_CHILD.to_string(),
|
||||
secret_key: "fixture-child-secret".to_string(),
|
||||
expiration: None,
|
||||
session_token: String::new(),
|
||||
status: "on".to_string(),
|
||||
parent_user: IDENTITY_PARENT.to_string(),
|
||||
claims: Some(HashMap::from([(
|
||||
IAM_POLICY_CLAIM_NAME_SA.to_string(),
|
||||
serde_json::json!("inherited-policy"),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(child.credentials.is_service_account(), "child fixture must be a service account");
|
||||
store
|
||||
.save_user_identity(IDENTITY_PARENT, UserType::Reg, parent, None)
|
||||
.await
|
||||
.expect("persist parent identity");
|
||||
store
|
||||
.save_user_identity(IDENTITY_CHILD, UserType::Svc, child, None)
|
||||
.await
|
||||
.expect("persist service child identity");
|
||||
let manager = IamCache {
|
||||
cache: Cache::default(),
|
||||
api: store,
|
||||
state: Arc::new(AtomicU8::new(IamState::Ready as u8)),
|
||||
loading: Arc::new(AtomicBool::new(false)),
|
||||
roles: HashMap::new(),
|
||||
send_chan: tokio::sync::mpsc::channel::<i64>(1).0,
|
||||
last_timestamp: AtomicI64::new(0),
|
||||
sync_failures: AtomicU64::new(0),
|
||||
sync_successes: AtomicU64::new(0),
|
||||
last_sync_duration_millis: AtomicU64::new(0),
|
||||
};
|
||||
manager
|
||||
.load_user(IDENTITY_PARENT)
|
||||
.await
|
||||
.expect("load persisted parent into cache");
|
||||
manager
|
||||
.load_user(IDENTITY_CHILD)
|
||||
.await
|
||||
.expect("load persisted service child into cache");
|
||||
Self {
|
||||
previous_store,
|
||||
ecstore,
|
||||
manager,
|
||||
_route_guard: route_guard,
|
||||
_temp_dir: temp_dir,
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_parent_and_child(&self) {
|
||||
let parent = self
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect("read parent from real storage");
|
||||
assert_eq!(parent.credentials.access_key, IDENTITY_PARENT);
|
||||
assert_eq!(parent.credentials.parent_user, "");
|
||||
let child = self
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_CHILD, UserType::Svc)
|
||||
.await
|
||||
.expect("read child from real storage");
|
||||
assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
|
||||
assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
|
||||
assert!(child.credentials.is_service_account());
|
||||
let cached_parent = self.manager.get_user(IDENTITY_PARENT).await.expect("parent remains in cache");
|
||||
assert_eq!(cached_parent.credentials.access_key, IDENTITY_PARENT);
|
||||
assert_eq!(cached_parent.credentials.parent_user, "");
|
||||
let cached_child = self
|
||||
.manager
|
||||
.get_user(IDENTITY_CHILD)
|
||||
.await
|
||||
.expect("service child remains in cache");
|
||||
assert_eq!(cached_child.credentials.access_key, IDENTITY_CHILD);
|
||||
assert_eq!(cached_child.credentials.parent_user, IDENTITY_PARENT);
|
||||
assert!(cached_child.credentials.is_service_account());
|
||||
}
|
||||
|
||||
async fn overwrite_parent_bytes(&self, bytes: &[u8]) {
|
||||
let path = format!("{}{IDENTITY_PARENT}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
|
||||
save_config(self.ecstore.clone(), &path, bytes.to_vec())
|
||||
.await
|
||||
.expect("overwrite real parent identity bytes");
|
||||
}
|
||||
|
||||
async fn assert_child_storage_and_identity_cache(&self) {
|
||||
let child = self
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_CHILD, UserType::Svc)
|
||||
.await
|
||||
.expect("notification must preserve persisted service child");
|
||||
assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
|
||||
assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
|
||||
assert!(child.credentials.is_service_account());
|
||||
let parent = self
|
||||
.manager
|
||||
.get_user(IDENTITY_PARENT)
|
||||
.await
|
||||
.expect("failed notification preserves cached parent");
|
||||
assert_eq!(parent.credentials.access_key, IDENTITY_PARENT);
|
||||
assert_eq!(parent.credentials.parent_user, "");
|
||||
let child = self
|
||||
.manager
|
||||
.get_user(IDENTITY_CHILD)
|
||||
.await
|
||||
.expect("failed notification preserves cached child");
|
||||
assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
|
||||
assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
|
||||
assert!(child.credentials.is_service_account());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_store_fixture_roundtrips_parent_and_service_child() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
let before = fixture.previous_store.clone();
|
||||
let selected = object_store_handle().expect("fixture route selects its ECStore");
|
||||
assert!(Arc::ptr_eq(&selected, &fixture.ecstore));
|
||||
drop(selected);
|
||||
fixture.assert_parent_and_child().await;
|
||||
drop(fixture);
|
||||
assert!(TEST_STORE_ROUTE.lock().expect("inspect cleared test store route").is_none());
|
||||
match (before, object_store_handle()) {
|
||||
(None, None) => {}
|
||||
(Some(before), Some(after)) => assert!(Arc::ptr_eq(&before, &after), "fixture drop restores prior store identity"),
|
||||
_ => panic!("fixture drop must restore the prior store option"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_notification_valid_parent_preserves_child_storage_and_cache() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.assert_parent_and_child().await;
|
||||
fixture
|
||||
.manager
|
||||
.user_notification_handler(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect("refresh valid parent notification");
|
||||
fixture.assert_parent_and_child().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_store_fixture_raw_valid_overwrite_is_observed() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
let parent = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect("load valid parent before raw overwrite");
|
||||
let mut raw = serde_json::to_value(parent).expect("serialize valid parent fixture");
|
||||
raw["credentials"]["name"] = serde_json::json!("updated-name");
|
||||
let path = format!("{}{IDENTITY_PARENT}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
|
||||
save_config(
|
||||
fixture.ecstore.clone(),
|
||||
&path,
|
||||
serde_json::to_vec(&raw).expect("encode raw parent fixture"),
|
||||
)
|
||||
.await
|
||||
.expect("overwrite parent through real config storage");
|
||||
fixture
|
||||
.manager
|
||||
.load_user(IDENTITY_PARENT)
|
||||
.await
|
||||
.expect("reload parent after raw overwrite");
|
||||
let stored = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect("read overwritten parent from storage");
|
||||
assert_eq!(stored.credentials.name.as_deref(), Some("updated-name"));
|
||||
let cached = fixture
|
||||
.manager
|
||||
.get_user(IDENTITY_PARENT)
|
||||
.await
|
||||
.expect("read overwritten parent from cache");
|
||||
assert_eq!(cached.credentials.name.as_deref(), Some("updated-name"));
|
||||
fixture.assert_parent_and_child().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_corrupt_reads_preserve_crypto_error() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.overwrite_parent_bytes(&[0xff, 0xfe, 0xfd]).await;
|
||||
let locked = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect_err("corrupt stored identity must fail locked read");
|
||||
assert!(matches!(locked, IamError::CryptoError(_)), "preserve crypto error, got {locked:?}");
|
||||
let mut users = HashMap::new();
|
||||
let no_lock = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
|
||||
.await
|
||||
.expect_err("corrupt stored identity must fail no-lock read");
|
||||
assert!(matches!(no_lock, IamError::CryptoError(_)), "preserve crypto error, got {no_lock:?}");
|
||||
assert!(users.is_empty(), "failed no-lock read must not populate users");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_corrupt_notification_preserves_child_storage_and_cache() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.assert_parent_and_child().await;
|
||||
fixture.overwrite_parent_bytes(&[0xff, 0xfe, 0xfd]).await;
|
||||
let notification = fixture
|
||||
.manager
|
||||
.user_notification_handler(IDENTITY_PARENT, UserType::Reg)
|
||||
.await;
|
||||
fixture.assert_child_storage_and_identity_cache().await;
|
||||
let error = notification.expect_err("corrupt parent notification must return an error");
|
||||
assert!(
|
||||
matches!(error, IamError::CryptoError(_)),
|
||||
"preserve notification crypto error, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_invalid_schema_preserves_decode_error() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.assert_parent_and_child().await;
|
||||
fixture
|
||||
.overwrite_parent_bytes(br#"{"version":"not-an-integer","credentials":{}}"#)
|
||||
.await;
|
||||
let error = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect_err("invalid identity field type must fail decode");
|
||||
assert_identity_data_error(error);
|
||||
let mut users = HashMap::new();
|
||||
let error = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
|
||||
.await
|
||||
.expect_err("invalid identity field type must fail no-lock decode");
|
||||
assert_identity_data_error(error);
|
||||
assert!(users.is_empty(), "failed decode must not populate users");
|
||||
let notification = fixture
|
||||
.manager
|
||||
.user_notification_handler(IDENTITY_PARENT, UserType::Reg)
|
||||
.await;
|
||||
fixture.assert_child_storage_and_identity_cache().await;
|
||||
assert_identity_data_error(notification.expect_err("invalid parent notification must return decode error"));
|
||||
}
|
||||
|
||||
fn assert_identity_data_error(error: IamError) {
|
||||
let IamError::Io(inner) = error else {
|
||||
panic!("identity decode must preserve Io serde error, got {error:?}");
|
||||
};
|
||||
let cause = inner
|
||||
.get_ref()
|
||||
.and_then(|cause| cause.downcast_ref::<serde_json::Error>())
|
||||
.expect("identity decode error retains serde source");
|
||||
assert!(cause.is_data(), "identity field type must produce serde Data error");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_physically_missing_is_no_such_user() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
let name = "physically-missing-identity";
|
||||
let error = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(name, UserType::Reg)
|
||||
.await
|
||||
.expect_err("missing identity must be absent");
|
||||
assert!(
|
||||
matches!(error, IamError::NoSuchUser(ref missing) if missing == name),
|
||||
"preserve absent name, got {error:?}"
|
||||
);
|
||||
let mut users = HashMap::new();
|
||||
let error = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_no_lock(name, UserType::Reg, &mut users)
|
||||
.await
|
||||
.expect_err("missing no-lock identity must be absent");
|
||||
assert!(
|
||||
matches!(error, IamError::NoSuchUser(ref missing) if missing == name),
|
||||
"preserve absent name, got {error:?}"
|
||||
);
|
||||
assert!(users.is_empty(), "missing read must not populate users");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_empty_reads_preserve_crypto_error() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.overwrite_parent_bytes(&[]).await;
|
||||
let locked = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect_err("empty stored identity must fail locked read");
|
||||
assert!(matches!(locked, IamError::CryptoError(_)), "preserve crypto error, got {locked:?}");
|
||||
let mut users = HashMap::new();
|
||||
let no_lock = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
|
||||
.await
|
||||
.expect_err("empty stored identity must fail no-lock read");
|
||||
assert!(matches!(no_lock, IamError::CryptoError(_)), "preserve crypto error, got {no_lock:?}");
|
||||
assert!(users.is_empty(), "failed no-lock read must not populate users");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_empty_notification_preserves_child_storage_and_cache() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.assert_parent_and_child().await;
|
||||
fixture.overwrite_parent_bytes(&[]).await;
|
||||
let notification = fixture
|
||||
.manager
|
||||
.user_notification_handler(IDENTITY_PARENT, UserType::Reg)
|
||||
.await;
|
||||
fixture.assert_child_storage_and_identity_cache().await;
|
||||
let error = notification.expect_err("empty parent notification must return an error");
|
||||
assert!(
|
||||
matches!(error, IamError::CryptoError(_)),
|
||||
"preserve notification crypto error, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn policy_empty_payload_retains_no_such_policy() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
let name = "empty-policy";
|
||||
let path = format!("{}{name}/policy.json", IAM_CONFIG_POLICIES_PREFIX.as_str());
|
||||
save_config(fixture.ecstore.clone(), &path, Vec::new())
|
||||
.await
|
||||
.expect("save real empty policy payload");
|
||||
let locked = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_policy(name)
|
||||
.await
|
||||
.err()
|
||||
.expect("empty policy must retain absent-policy behavior");
|
||||
assert!(matches!(locked, IamError::NoSuchPolicy), "preserve policy error, got {locked:?}");
|
||||
let mut policies = HashMap::new();
|
||||
let no_lock = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_policy_doc_no_lock(name, &mut policies)
|
||||
.await
|
||||
.expect_err("empty no-lock policy must retain absent-policy behavior");
|
||||
assert!(matches!(no_lock, IamError::NoSuchPolicy), "preserve policy error, got {no_lock:?}");
|
||||
assert!(policies.is_empty(), "failed no-lock policy read must not populate policies");
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }
|
||||
time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] }
|
||||
serde = { workspace = true, features = ["derive", "rc"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
uuid = { workspace = true, features = ["serde"] }
|
||||
thiserror.workspace = true
|
||||
strum = { workspace = true, features = ["derive"] }
|
||||
rustfs-crypto = { workspace = true }
|
||||
|
||||
@@ -22,6 +22,33 @@ use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// A nonnil stored principal identifier; parsing it does not establish identity authority.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(try_from = "uuid::Uuid", into = "uuid::Uuid")]
|
||||
pub struct PrincipalIncarnation(uuid::Uuid);
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("principal incarnation must not be nil")]
|
||||
pub struct InvalidPrincipalIncarnation;
|
||||
|
||||
impl TryFrom<uuid::Uuid> for PrincipalIncarnation {
|
||||
type Error = InvalidPrincipalIncarnation;
|
||||
|
||||
fn try_from(value: uuid::Uuid) -> Result<Self, Self::Error> {
|
||||
if value.is_nil() {
|
||||
Err(InvalidPrincipalIncarnation)
|
||||
} else {
|
||||
Ok(Self(value))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<PrincipalIncarnation> for uuid::Uuid {
|
||||
fn from(value: PrincipalIncarnation) -> Self {
|
||||
value.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
|
||||
pub struct UserIdentity {
|
||||
pub version: i64,
|
||||
@@ -29,6 +56,8 @@ pub struct UserIdentity {
|
||||
/// updatedAt (RFC3339), legacy RustFS: update_at. Serialize as updatedAt
|
||||
#[serde(rename = "updatedAt", alias = "update_at", default, with = "crate::serde_datetime::option")]
|
||||
pub update_at: Option<OffsetDateTime>,
|
||||
#[serde(default, rename = "principalIncarnation", skip_serializing_if = "Option::is_none")]
|
||||
pub principal_incarnation: Option<PrincipalIncarnation>,
|
||||
}
|
||||
|
||||
impl UserIdentity {
|
||||
@@ -44,6 +73,7 @@ impl UserIdentity {
|
||||
version: 1,
|
||||
credentials,
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
principal_incarnation: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,13 +103,124 @@ impl From<Credentials> for UserIdentity {
|
||||
version: 1,
|
||||
credentials: value,
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
principal_incarnation: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::UserIdentity;
|
||||
use super::{PrincipalIncarnation, UserIdentity};
|
||||
use rustfs_credentials::Credentials;
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
const INCARNATION: &str = "8f422eba-aab4-4ec3-a7ec-d1717463b520";
|
||||
|
||||
#[test]
|
||||
fn test_user_identity_legacy_fields_absent() {
|
||||
let mut legacy = json!({"version": 1, "credentials": {"accessKey": "ak", "secretKey": "sk12345678"}});
|
||||
let identity: UserIdentity = serde_json::from_value(legacy.clone()).expect("decode legacy identity");
|
||||
assert!(identity.principal_incarnation.is_none());
|
||||
legacy["principalIncarnation"] = serde_json::Value::Null;
|
||||
let identity: UserIdentity = serde_json::from_value(legacy).expect("decode null incarnation");
|
||||
assert!(identity.principal_incarnation.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_user_identity_legacy_serialization_shape() {
|
||||
let identity = UserIdentity::default();
|
||||
let serialized = serde_json::to_value(&identity).expect("serialize legacy identity");
|
||||
assert_eq!(serialized, json!({"version": 0, "credentials": identity.credentials, "updatedAt": null}));
|
||||
assert!(serialized.get("principalIncarnation").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_user_identity_constructors_do_not_mint_incarnation() {
|
||||
let default = UserIdentity::default();
|
||||
assert!(default.principal_incarnation.is_none());
|
||||
assert_eq!(default.version, 0);
|
||||
assert!(default.update_at.is_none());
|
||||
let credentials = Credentials {
|
||||
access_key: "ak".to_string(),
|
||||
secret_key: "sk12345678".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let expected_credentials = serde_json::to_value(&credentials).expect("serialize expected credentials");
|
||||
for identity in [UserIdentity::new(credentials.clone()), UserIdentity::from(credentials)] {
|
||||
assert!(identity.principal_incarnation.is_none());
|
||||
assert_eq!(identity.version, 1);
|
||||
assert!(identity.update_at.is_some());
|
||||
assert_eq!(
|
||||
serde_json::to_value(&identity.credentials).expect("serialize constructor credentials"),
|
||||
expected_credentials
|
||||
);
|
||||
assert!(
|
||||
serde_json::to_value(identity)
|
||||
.expect("serialize constructor identity")
|
||||
.get("principalIncarnation")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_principal_incarnation_rejects_nil_and_malformed() {
|
||||
assert!(PrincipalIncarnation::try_from(Uuid::nil()).is_err());
|
||||
for value in ["00000000-0000-0000-0000-000000000000", "not-a-uuid", ""] {
|
||||
assert!(serde_json::from_value::<PrincipalIncarnation>(json!(value)).is_err(), "reject {value}");
|
||||
let identity = json!({"version": 1, "credentials": {}, "principalIncarnation": value});
|
||||
assert!(serde_json::from_value::<UserIdentity>(identity).is_err(), "identity rejects {value}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_principal_incarnation_roundtrip_preserves_uuid() {
|
||||
let uuid = Uuid::parse_str(INCARNATION).expect("parse fixed incarnation");
|
||||
let incarnation = PrincipalIncarnation::try_from(uuid).expect("accept nonnil incarnation");
|
||||
assert_eq!(Uuid::from(incarnation).as_bytes(), uuid.as_bytes());
|
||||
let serialized = serde_json::to_value(incarnation).expect("serialize incarnation");
|
||||
assert_eq!(serialized, json!(INCARNATION));
|
||||
assert_eq!(
|
||||
serde_json::from_value::<PrincipalIncarnation>(serialized).expect("decode incarnation"),
|
||||
incarnation
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_user_identity_live_incarnation_roundtrip() {
|
||||
let mut identity = UserIdentity::new(Credentials::default());
|
||||
identity.principal_incarnation = Some(
|
||||
PrincipalIncarnation::try_from(Uuid::parse_str(INCARNATION).expect("parse fixed incarnation"))
|
||||
.expect("accept fixed incarnation"),
|
||||
);
|
||||
let serialized = serde_json::to_value(&identity).expect("serialize identity with incarnation");
|
||||
assert_eq!(serialized["principalIncarnation"], INCARNATION);
|
||||
let encoded = serde_json::to_string(&identity).expect("encode identity with incarnation");
|
||||
let decoded: UserIdentity = serde_json::from_str(&encoded).expect("decode identity with incarnation");
|
||||
assert_eq!(decoded.principal_incarnation, identity.principal_incarnation);
|
||||
assert_eq!(serde_json::to_value(decoded).expect("reserialize identity"), serialized);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_user_identity_timestamp_aliases_unchanged() {
|
||||
let mut minio = json!({"version": 1, "credentials": {}, "updatedAt": "2025-03-07T12:00:00Z"});
|
||||
let encoded = serde_json::to_string(&minio).expect("encode updatedAt fixture");
|
||||
let identity: UserIdentity = serde_json::from_str(&encoded).expect("decode updatedAt timestamp");
|
||||
minio["update_at"] = minio
|
||||
.as_object_mut()
|
||||
.expect("identity object")
|
||||
.remove("updatedAt")
|
||||
.expect("existing timestamp");
|
||||
let encoded = serde_json::to_string(&minio).expect("encode update_at fixture");
|
||||
let legacy: UserIdentity = serde_json::from_str(&encoded).expect("decode update_at alias");
|
||||
assert_eq!(legacy.update_at, identity.update_at);
|
||||
assert_eq!(legacy.version, 1);
|
||||
let serialized = serde_json::to_value(legacy).expect("serialize legacy timestamp");
|
||||
assert_eq!(serialized["updatedAt"], "2025-03-07T12:00:00Z");
|
||||
assert!(serialized.get("update_at").is_none());
|
||||
assert!(serialized.get("principalIncarnation").is_none());
|
||||
}
|
||||
|
||||
/// Deserialize UserIdentity from MinIO-style JSON (RFC3339 updatedAt).
|
||||
#[test]
|
||||
|
||||
@@ -568,6 +568,42 @@ pub enum AdminAction {
|
||||
GetTableLifecycleAction,
|
||||
#[strum(serialize = "admin:SetTableLifecycle")]
|
||||
SetTableLifecycleAction,
|
||||
#[strum(serialize = "admin:CreatePluginDelegation")]
|
||||
CreatePluginDelegationAction,
|
||||
#[strum(serialize = "admin:IssuePluginAttempt")]
|
||||
IssuePluginAttemptAction,
|
||||
#[strum(serialize = "admin:GetPluginArtifactBucket")]
|
||||
GetPluginArtifactBucketAction,
|
||||
#[strum(serialize = "admin:CreatePluginArtifactBucket")]
|
||||
CreatePluginArtifactBucketAction,
|
||||
#[strum(serialize = "admin:SealPluginArtifactAttempt")]
|
||||
SealPluginArtifactAttemptAction,
|
||||
#[strum(serialize = "admin:SealPluginArtifactRepairs")]
|
||||
SealPluginArtifactRepairsAction,
|
||||
#[strum(serialize = "admin:SealPluginArtifactWrites")]
|
||||
SealPluginArtifactWritesAction,
|
||||
#[strum(serialize = "admin:InventoryPluginArtifactAttempt")]
|
||||
InventoryPluginArtifactAttemptAction,
|
||||
#[strum(serialize = "admin:RetirePluginArtifactVersion")]
|
||||
RetirePluginArtifactVersionAction,
|
||||
#[strum(serialize = "admin:DeletePluginArtifactVersion")]
|
||||
DeletePluginArtifactVersionAction,
|
||||
#[strum(serialize = "admin:RetainPluginArtifactBackups")]
|
||||
RetainPluginArtifactBackupsAction,
|
||||
#[strum(serialize = "admin:SubmitPluginJob")]
|
||||
SubmitPluginJobAction,
|
||||
#[strum(serialize = "admin:GetPluginRun")]
|
||||
GetPluginRunAction,
|
||||
#[strum(serialize = "admin:CancelPluginRun")]
|
||||
CancelPluginRunAction,
|
||||
#[strum(serialize = "admin:ListPluginRuns")]
|
||||
ListPluginRunsAction,
|
||||
#[strum(serialize = "admin:ReplayPluginRun")]
|
||||
ReplayPluginRunAction,
|
||||
#[strum(serialize = "admin:GetPluginDelegation")]
|
||||
GetPluginDelegationAction,
|
||||
#[strum(serialize = "admin:RevokePluginDelegation")]
|
||||
RevokePluginDelegationAction,
|
||||
#[strum(serialize = "admin:GetTableCredentials")]
|
||||
GetTableCredentialsAction,
|
||||
#[strum(serialize = "admin:RunTableMaintenance")]
|
||||
@@ -601,6 +637,21 @@ pub enum AdminAction {
|
||||
}
|
||||
|
||||
impl AdminAction {
|
||||
pub(crate) fn is_artifact_resource_scoped(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::CreatePluginArtifactBucketAction
|
||||
| Self::GetPluginArtifactBucketAction
|
||||
| Self::SealPluginArtifactAttemptAction
|
||||
| Self::SealPluginArtifactRepairsAction
|
||||
| Self::SealPluginArtifactWritesAction
|
||||
| Self::InventoryPluginArtifactAttemptAction
|
||||
| Self::RetirePluginArtifactVersionAction
|
||||
| Self::DeletePluginArtifactVersionAction
|
||||
| Self::RetainPluginArtifactBackupsAction
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_table_resource_scoped(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
@@ -713,6 +764,24 @@ impl AdminAction {
|
||||
| AdminAction::GetTableLifecycleAction
|
||||
| AdminAction::SetTableLifecycleAction
|
||||
| AdminAction::GetTableCredentialsAction
|
||||
| AdminAction::IssuePluginAttemptAction
|
||||
| AdminAction::CreatePluginArtifactBucketAction
|
||||
| AdminAction::GetPluginArtifactBucketAction
|
||||
| AdminAction::SealPluginArtifactAttemptAction
|
||||
| AdminAction::SealPluginArtifactRepairsAction
|
||||
| AdminAction::SealPluginArtifactWritesAction
|
||||
| AdminAction::InventoryPluginArtifactAttemptAction
|
||||
| AdminAction::RetirePluginArtifactVersionAction
|
||||
| AdminAction::DeletePluginArtifactVersionAction
|
||||
| AdminAction::RetainPluginArtifactBackupsAction
|
||||
| AdminAction::SubmitPluginJobAction
|
||||
| AdminAction::GetPluginRunAction
|
||||
| AdminAction::CancelPluginRunAction
|
||||
| AdminAction::ListPluginRunsAction
|
||||
| AdminAction::ReplayPluginRunAction
|
||||
| AdminAction::GetPluginDelegationAction
|
||||
| AdminAction::CreatePluginDelegationAction
|
||||
| AdminAction::RevokePluginDelegationAction
|
||||
| AdminAction::RunTableMaintenanceAction
|
||||
| AdminAction::GetTableMetadataLocationAction
|
||||
| AdminAction::SetTableMetadataLocationAction
|
||||
@@ -913,6 +982,35 @@ mod tests {
|
||||
assert_eq!(arr[0].as_str().unwrap(), "s3:*");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plugin_delegation_actions_are_explicit_and_unscoped() {
|
||||
for (name, expected) in [
|
||||
("admin:GetPluginDelegation", AdminAction::GetPluginDelegationAction),
|
||||
("admin:CreatePluginDelegation", AdminAction::CreatePluginDelegationAction),
|
||||
("admin:RevokePluginDelegation", AdminAction::RevokePluginDelegationAction),
|
||||
("admin:SubmitPluginJob", AdminAction::SubmitPluginJobAction),
|
||||
("admin:GetPluginRun", AdminAction::GetPluginRunAction),
|
||||
("admin:CancelPluginRun", AdminAction::CancelPluginRunAction),
|
||||
("admin:ListPluginRuns", AdminAction::ListPluginRunsAction),
|
||||
("admin:ReplayPluginRun", AdminAction::ReplayPluginRunAction),
|
||||
] {
|
||||
assert_eq!(AdminAction::try_from(name).unwrap(), expected);
|
||||
assert!(expected.is_valid());
|
||||
assert!(!expected.is_table_resource_scoped());
|
||||
}
|
||||
assert_eq!(
|
||||
AdminAction::try_from("admin:IssuePluginAttempt").unwrap(),
|
||||
AdminAction::IssuePluginAttemptAction
|
||||
);
|
||||
assert!(AdminAction::IssuePluginAttemptAction.is_valid());
|
||||
assert!(AdminAction::SubmitPluginJobAction.is_valid());
|
||||
assert!(AdminAction::GetPluginRunAction.is_valid());
|
||||
assert!(AdminAction::CancelPluginRunAction.is_valid());
|
||||
assert!(AdminAction::ListPluginRunsAction.is_valid());
|
||||
assert!(AdminAction::ReplayPluginRunAction.is_valid());
|
||||
assert!(AdminAction::try_from("admin:IssuePluginCredentials").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_replication_metrics_admin_action_is_valid() {
|
||||
assert!(AdminAction::GetReplicationMetricsAction.is_valid());
|
||||
@@ -1072,6 +1170,21 @@ mod tests {
|
||||
assert!(set_action.is_valid());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_backup_retention_requires_resource_scoped_admin_action() {
|
||||
let action = AdminAction::try_from("admin:RetainPluginArtifactBackups").expect("retention action recognized");
|
||||
assert!(action.is_valid(), "O02P3_RETENTION_ACTION_VALID");
|
||||
assert!(action.is_artifact_resource_scoped(), "O02P3_RETENTION_ACTION_SCOPED");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_delete_requires_resource_scoped_admin_action() {
|
||||
let action = AdminAction::try_from("admin:DeletePluginArtifactVersion")
|
||||
.expect("A03GC2_INVARIANT dedicated delete action is recognized");
|
||||
assert!(action.is_valid());
|
||||
assert!(action.is_artifact_resource_scoped(), "A03GC2_INVARIANT explicit object scope");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_table_metadata_admin_actions_are_valid() {
|
||||
let get_action = AdminAction::try_from("admin:GetTableMetadata").expect("Should parse GetTableMetadata action");
|
||||
|
||||
@@ -99,7 +99,7 @@ impl Statement {
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(args.action, Action::AdminAction(action) if action.is_table_resource_scoped())
|
||||
!matches!(args.action, Action::AdminAction(action) if action.is_table_resource_scoped() || action.is_artifact_resource_scoped())
|
||||
}
|
||||
|
||||
fn is_kms(&self) -> bool {
|
||||
@@ -265,6 +265,16 @@ impl Statement {
|
||||
return self.kms_key_scope_matches(args, resolver).await;
|
||||
}
|
||||
|
||||
// New artifact controls require an explicit Allow resource. Preserve
|
||||
// unscoped Deny (including admin:*) so introducing scoping cannot weaken it.
|
||||
if matches!(args.action, Action::AdminAction(action) if action.is_artifact_resource_scoped())
|
||||
&& !deny
|
||||
&& self.resources.is_empty()
|
||||
&& self.not_resources.is_empty()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let resource = build_resource(
|
||||
&args.action,
|
||||
args.bucket,
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
use rustfs_policy::policy::{
|
||||
Args, Policy,
|
||||
action::{Action, AdminAction},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn allowed(statements: Value, action: &str, bucket: &str, object: &str) -> bool {
|
||||
let policy: Policy =
|
||||
serde_json::from_value(json!({"Version":"2012-10-17","Statement":statements})).expect("valid artifact IAM policy");
|
||||
pollster::block_on(policy.is_allowed(&Args {
|
||||
account: "controller",
|
||||
groups: &None,
|
||||
action: Action::try_from(action).expect("registered artifact action"),
|
||||
bucket,
|
||||
object,
|
||||
conditions: &HashMap::new(),
|
||||
claims: &HashMap::new(),
|
||||
is_owner: false,
|
||||
deny_only: false,
|
||||
}))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_resource_allow_is_scoped_to_bucket_and_attempt() {
|
||||
for action in [
|
||||
"admin:SealPluginArtifactAttempt",
|
||||
"admin:SealPluginArtifactRepairs",
|
||||
"admin:RetirePluginArtifactVersion",
|
||||
] {
|
||||
let policy = json!([{"Effect":"Allow","Action":action,"Resource":["arn:aws:s3:::artifacts/out/attempt-a", "arn:aws:s3:::artifacts/out/attempt-a/*"]}]);
|
||||
assert!(allowed(policy.clone(), action, "artifacts", "out/attempt-a/"));
|
||||
assert!(allowed(policy.clone(), action, "artifacts", "out/attempt-a/image.png"));
|
||||
assert!(
|
||||
!allowed(policy.clone(), action, "other", "out/attempt-a/image.png"),
|
||||
"ARTIFACT_RESOURCE_BUCKET"
|
||||
);
|
||||
assert!(!allowed(policy, action, "artifacts", "out/attempt-b/"), "ARTIFACT_RESOURCE_ATTEMPT");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_resource_deny_and_not_resource_are_enforced() {
|
||||
for action in [
|
||||
"admin:SealPluginArtifactAttempt",
|
||||
"admin:SealPluginArtifactRepairs",
|
||||
"admin:RetirePluginArtifactVersion",
|
||||
] {
|
||||
let scoped_allow = json!({"Effect":"Allow","Action":action,"Resource":"arn:aws:s3:::artifacts/out/*"});
|
||||
let explicit_deny = json!({"Effect":"Deny","Action":action,"Resource":"arn:aws:s3:::artifacts/out/private/*"});
|
||||
assert!(!allowed(
|
||||
json!([scoped_allow.clone(), explicit_deny.clone()]),
|
||||
action,
|
||||
"artifacts",
|
||||
"out/private/image"
|
||||
));
|
||||
assert!(allowed(
|
||||
json!([scoped_allow.clone(), explicit_deny]),
|
||||
action,
|
||||
"artifacts",
|
||||
"out/public/image"
|
||||
));
|
||||
let except_public = json!({"Effect":"Deny","Action":action,"NotResource":"arn:aws:s3:::artifacts/out/public/*"});
|
||||
assert!(allowed(
|
||||
json!([scoped_allow.clone(), except_public.clone()]),
|
||||
action,
|
||||
"artifacts",
|
||||
"out/public/image"
|
||||
));
|
||||
assert!(!allowed(
|
||||
json!([scoped_allow.clone(), except_public]),
|
||||
action,
|
||||
"artifacts",
|
||||
"out/private/image"
|
||||
));
|
||||
let allow_except_private =
|
||||
json!([{"Effect":"Allow","Action":action,"NotResource":"arn:aws:s3:::artifacts/out/private/*"}]);
|
||||
assert!(!allowed(allow_except_private.clone(), action, "artifacts", "out/private/image"));
|
||||
assert!(allowed(allow_except_private, action, "artifacts", "out/public/image"));
|
||||
assert!(
|
||||
!allowed(
|
||||
json!([scoped_allow, {"Effect":"Deny","Action":"admin:*"}]),
|
||||
action,
|
||||
"artifacts",
|
||||
"out/public/image"
|
||||
),
|
||||
"ARTIFACT_GLOBAL_DENY"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_resource_empty_allow_is_rejected_without_changing_old_admin() {
|
||||
for action in [
|
||||
"admin:SealPluginArtifactAttempt",
|
||||
"admin:SealPluginArtifactRepairs",
|
||||
"admin:RetirePluginArtifactVersion",
|
||||
] {
|
||||
for granted in [action, "admin:*"] {
|
||||
assert!(
|
||||
!allowed(json!([{"Effect":"Allow","Action":granted}]), action, "artifacts", "out/"),
|
||||
"ARTIFACT_RESOURCE_REQUIRED"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(allowed(
|
||||
json!([{"Effect":"Allow","Action":"admin:IssuePluginAttempt"}]),
|
||||
"admin:IssuePluginAttempt",
|
||||
"",
|
||||
""
|
||||
));
|
||||
assert!(allowed(
|
||||
json!([{"Effect":"Allow","Action":"admin:IssuePluginAttempt","Resource":"arn:aws:s3:::unrelated/*"}]),
|
||||
"admin:IssuePluginAttempt",
|
||||
"",
|
||||
""
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_resource_single_object_does_not_authorize_attempt_seal() {
|
||||
let action = "admin:SealPluginArtifactAttempt";
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow","Action":action,"Resource":"arn:aws:s3:::artifacts/out/attempt-a/image.png"}]),
|
||||
action,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
assert!(allowed(
|
||||
json!([{"Effect":"Allow","Action":action,"Resource":"arn:aws:s3:::artifacts/out/attempt-a"}]),
|
||||
action,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_bucket_create_resource_requires_exact_bucket_authority() {
|
||||
let action = "admin:CreatePluginArtifactBucket";
|
||||
let scoped = json!({"Effect":"Allow","Action":action,"Resource":"arn:aws:s3:::artifacts"});
|
||||
assert!(allowed(json!([scoped]), action, "artifacts", ""));
|
||||
assert!(!allowed(json!([scoped]), action, "another", ""), "ARTIFACT_CREATE_RESOURCE");
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow","Action":action,"Resource":"arn:aws:s3:::artifacts/*"}]),
|
||||
action,
|
||||
"artifacts",
|
||||
""
|
||||
));
|
||||
assert!(!allowed(json!([{"Effect":"Allow","Action":action}]), action, "artifacts", ""));
|
||||
assert!(!allowed(json!([scoped, {"Effect":"Deny","Action":"admin:*"}]), action, "artifacts", ""));
|
||||
assert!(!allowed(
|
||||
json!([scoped, {"Effect":"Deny","Action":action,"Resource":"arn:aws:s3:::artifacts"}]),
|
||||
action,
|
||||
"artifacts",
|
||||
""
|
||||
));
|
||||
let not_resource = json!([{"Effect":"Allow","Action":action,"NotResource":"arn:aws:s3:::artifacts"}]);
|
||||
assert!(!allowed(not_resource.clone(), action, "artifacts", ""));
|
||||
assert!(allowed(not_resource, action, "another", ""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_bucket_inspect_resource_requires_independent_bucket_authority() {
|
||||
let action = "admin:GetPluginArtifactBucket";
|
||||
let scoped = json!({"Effect":"Allow","Action":action,"Resource":"arn:aws:s3:::artifacts"});
|
||||
assert!(allowed(json!([scoped]), action, "artifacts", ""));
|
||||
assert!(!allowed(json!([scoped]), action, "another", ""), "ARTIFACT_INSPECT_RESOURCE");
|
||||
for grant in [
|
||||
json!({"Effect":"Allow","Action":action}),
|
||||
json!({"Effect":"Allow","Action":action,"Resource":"arn:aws:s3:::artifacts/*"}),
|
||||
json!({"Effect":"Allow","Action":"admin:CreatePluginArtifactBucket","Resource":"arn:aws:s3:::artifacts"}),
|
||||
] {
|
||||
assert!(!allowed(json!([grant]), action, "artifacts", ""));
|
||||
}
|
||||
assert!(!allowed(json!([scoped, {"Effect":"Deny","Action":"admin:*"}]), action, "artifacts", ""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_repair_seal_action_is_independent_and_round_trips() {
|
||||
assert!(AdminAction::SealPluginArtifactRepairsAction.is_valid());
|
||||
let action = Action::AdminAction(AdminAction::SealPluginArtifactRepairsAction);
|
||||
let wire = "admin:SealPluginArtifactRepairs";
|
||||
assert_eq!(Action::try_from(wire).expect("repair seal action"), action);
|
||||
assert_eq!(serde_json::to_value(action).expect("serialized repair seal action"), json!(wire));
|
||||
for grant in ["admin:SealPluginArtifactAttempt", "admin:RetirePluginArtifactVersion"] {
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow","Action":wire,"Resource":"arn:aws:s3:::artifacts/out/attempt-a"}]),
|
||||
grant,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow","Action":grant,"Resource":"arn:aws:s3:::artifacts/out/attempt-a"}]),
|
||||
wire,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
}
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow","Action":wire,"Resource":"arn:aws:s3:::artifacts/out/attempt-a/image.png"}]),
|
||||
wire,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_writes_seal_requires_independent_explicit_attempt_authority() {
|
||||
let wire = "admin:SealPluginArtifactWrites";
|
||||
let action = Action::AdminAction(AdminAction::SealPluginArtifactWritesAction);
|
||||
assert!(AdminAction::SealPluginArtifactWritesAction.is_valid());
|
||||
assert_eq!(Action::try_from(wire).expect("writes seal action"), action);
|
||||
assert_eq!(serde_json::to_value(action).expect("writes seal wire form"), json!(wire));
|
||||
let grant = json!({"Effect":"Allow", "Action":wire, "Resource":"arn:aws:s3:::artifacts/out/attempt-a"});
|
||||
assert!(allowed(json!([grant]), wire, "artifacts", "out/attempt-a/"));
|
||||
assert!(!allowed(json!([grant]), wire, "another", "out/attempt-a/"));
|
||||
assert!(!allowed(json!([grant]), wire, "artifacts", "out/attempt-b/"));
|
||||
for resource in [None, Some("arn:aws:s3:::artifacts/out/attempt-a/object")] {
|
||||
let mut unscoped = json!({"Effect":"Allow", "Action":wire});
|
||||
if let Some(resource) = resource {
|
||||
unscoped["Resource"] = json!(resource);
|
||||
}
|
||||
assert!(!allowed(json!([unscoped]), wire, "artifacts", "out/attempt-a/"));
|
||||
}
|
||||
assert!(!allowed(
|
||||
json!([grant, {"Effect":"Deny", "Action":"admin:*"}]),
|
||||
wire,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
for old in ["admin:SealPluginArtifactAttempt", "admin:SealPluginArtifactRepairs"] {
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow", "Action":old, "Resource":"arn:aws:s3:::artifacts/out/attempt-a"}]),
|
||||
wire,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow", "Action":wire, "Resource":"arn:aws:s3:::artifacts/out/attempt-a"}]),
|
||||
old,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
}
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow", "Action":["admin:SealPluginArtifactAttempt", "admin:SealPluginArtifactRepairs"], "Resource":"arn:aws:s3:::artifacts/out/attempt-a"}]),
|
||||
wire,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_inventory_resource_requires_independent_authority() {
|
||||
let wire = "admin:InventoryPluginArtifactAttempt";
|
||||
let action = Action::try_from(wire).expect("A03I3A_INVENTORY_ACTION_REGISTERED");
|
||||
assert_eq!(serde_json::to_value(action).expect("inventory wire action"), json!(wire));
|
||||
let grant = json!({"Effect":"Allow","Action":wire,"Resource":"arn:aws:s3:::artifacts/out/attempt-a"});
|
||||
assert!(allowed(json!([grant]), wire, "artifacts", "out/attempt-a/"));
|
||||
for (bucket, prefix) in [("another", "out/attempt-a/"), ("artifacts", "out/attempt-b/")] {
|
||||
assert!(!allowed(json!([grant.clone()]), wire, bucket, prefix));
|
||||
}
|
||||
for granted in [wire, "admin:*"] {
|
||||
assert!(
|
||||
!allowed(json!([{"Effect":"Allow","Action":granted}]), wire, "artifacts", "out/attempt-a/"),
|
||||
"A03I3A_RESOURCE_REQUIRED"
|
||||
);
|
||||
}
|
||||
for other in [
|
||||
"admin:SealPluginArtifactAttempt",
|
||||
"admin:SealPluginArtifactRepairs",
|
||||
"admin:SealPluginArtifactWrites",
|
||||
"s3:ListBucket",
|
||||
] {
|
||||
assert!(!allowed(
|
||||
json!([{"Effect":"Allow","Action":other,"Resource":"arn:aws:s3:::artifacts/out/attempt-a"}]),
|
||||
wire,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
assert!(!allowed(json!([grant.clone()]), other, "artifacts", "out/attempt-a/"));
|
||||
}
|
||||
assert!(!allowed(
|
||||
json!([grant, {"Effect":"Deny","Action":"admin:*"}]),
|
||||
wire,
|
||||
"artifacts",
|
||||
"out/attempt-a/"
|
||||
));
|
||||
}
|
||||
@@ -308,6 +308,7 @@ pub async fn create_container(account: &str, container: &str, credentials: &Cred
|
||||
.make_bucket(
|
||||
&bucket_name,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: None,
|
||||
force_create: false,
|
||||
lock_enabled: false,
|
||||
versioning_enabled: false,
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use crate::{
|
||||
CanonicalBodyBuilder,
|
||||
proto_gen::node_service::{ArtifactInventoryFrame, ArtifactInventoryRequest},
|
||||
};
|
||||
use prost::Message;
|
||||
use std::io;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const VERSION: u32 = 1;
|
||||
pub const MAX_FRAME_BYTES: usize = 64 * 1024;
|
||||
pub const MAX_FRAME_ENTRIES: usize = 256;
|
||||
pub const MAX_TIMEOUT_MILLIS: u64 = 300_000;
|
||||
|
||||
pub fn canonical_request(request: &ArtifactInventoryRequest) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-artifact-inventory-request-v1\0");
|
||||
body.push_u32(request.protocol_version);
|
||||
body.push_bytes(&request.invocation_id)?;
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_bytes(&request.expected_disk_id)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_bytes(&request.bucket_incarnation_id)?;
|
||||
body.push_str(&request.attempt_prefix)?;
|
||||
body.push_u64(request.timeout_millis);
|
||||
body.push_u32(u32::try_from(request.phase)?);
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_frame(
|
||||
request: &ArtifactInventoryRequest,
|
||||
frame: &ArtifactInventoryFrame,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-artifact-inventory-frame-v1\0");
|
||||
body.push_bytes(&canonical_request(request)?)?;
|
||||
let mut unsigned = frame.clone();
|
||||
unsigned.response_proof.clear();
|
||||
body.push_bytes(&unsigned.encode_to_vec())?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
impl crate::CanonicalMutationBody for ArtifactInventoryRequest {
|
||||
fn canonical_body(&self) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
canonical_request(self)
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(message: &'static str) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, message)
|
||||
}
|
||||
fn non_nil(bytes: &[u8]) -> bool {
|
||||
Uuid::from_slice(bytes).is_ok_and(|id| !id.is_nil())
|
||||
}
|
||||
|
||||
pub fn validate_request(request: &ArtifactInventoryRequest) -> io::Result<()> {
|
||||
if request.protocol_version != VERSION
|
||||
|| !non_nil(&request.invocation_id)
|
||||
|| !non_nil(&request.expected_disk_id)
|
||||
|| !non_nil(&request.bucket_incarnation_id)
|
||||
|| request.disk.is_empty()
|
||||
|| request.disk.len() > 4096
|
||||
|| request.bucket.is_empty()
|
||||
|| request.bucket.len() > 255
|
||||
|| request.attempt_prefix.len() >= 1024
|
||||
|| !request.attempt_prefix.ends_with('/')
|
||||
|| request.timeout_millis == 0
|
||||
|| request.timeout_millis > MAX_TIMEOUT_MILLIS
|
||||
|| !matches!(request.phase, 0 | 1)
|
||||
{
|
||||
return Err(invalid("invalid artifact inventory request"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One response stream, finalized only after authenticated terminal and clean EOF.
|
||||
pub struct StreamState {
|
||||
next: u64,
|
||||
entries: u64,
|
||||
terminal: bool,
|
||||
}
|
||||
impl Default for StreamState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
impl StreamState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
next: 0,
|
||||
entries: 0,
|
||||
terminal: false,
|
||||
}
|
||||
}
|
||||
pub fn accept(
|
||||
&mut self,
|
||||
request: &ArtifactInventoryRequest,
|
||||
frame: &ArtifactInventoryFrame,
|
||||
verify: impl FnOnce(&[u8], &[u8]) -> bool,
|
||||
) -> io::Result<()> {
|
||||
use crate::proto_gen::node_service::artifact_inventory_frame::Payload;
|
||||
validate_request(request)?;
|
||||
if self.terminal || frame.sequence != self.next || frame.encoded_len() > MAX_FRAME_BYTES {
|
||||
return Err(invalid("artifact inventory frame sequence or size mismatch"));
|
||||
}
|
||||
let canonical = canonical_frame(request, frame).map_err(io::Error::other)?;
|
||||
if !verify(&canonical, &frame.response_proof) {
|
||||
return Err(invalid("artifact inventory frame proof mismatch"));
|
||||
}
|
||||
match &frame.payload {
|
||||
Some(Payload::Entries(entries)) if request.phase == 0 => {
|
||||
if entries.versions.is_empty() || entries.versions.len() > MAX_FRAME_ENTRIES {
|
||||
return Err(invalid("artifact inventory entry count is invalid"));
|
||||
}
|
||||
for version in &entries.versions {
|
||||
if version.key.len() > 1024
|
||||
|| version.key.len() <= request.attempt_prefix.len()
|
||||
|| !version.key.starts_with(&request.attempt_prefix)
|
||||
|| !non_nil(&version.version_id)
|
||||
{
|
||||
return Err(invalid("artifact inventory version scope is invalid"));
|
||||
}
|
||||
}
|
||||
self.entries = self
|
||||
.entries
|
||||
.checked_add(u64::try_from(entries.versions.len()).map_err(io::Error::other)?)
|
||||
.ok_or_else(|| invalid("artifact inventory count overflow"))?;
|
||||
}
|
||||
Some(Payload::Complete(complete))
|
||||
if request.phase == 0 && complete.disk_id == request.expected_disk_id && complete.entries == self.entries =>
|
||||
{
|
||||
self.terminal = true
|
||||
}
|
||||
Some(Payload::Verified(verified)) if request.phase == 1 && verified.disk_id == request.expected_disk_id => {
|
||||
self.terminal = true
|
||||
}
|
||||
_ => return Err(invalid("artifact inventory terminal, phase or identity mismatch")),
|
||||
}
|
||||
self.next = self
|
||||
.next
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| invalid("artifact inventory sequence overflow"))?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn finish(&self) -> io::Result<()> {
|
||||
if self.terminal {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(invalid("artifact inventory stream is incomplete"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::proto_gen::node_service::{
|
||||
ArtifactInventoryComplete, ArtifactInventoryVerified, artifact_inventory_frame::Payload,
|
||||
};
|
||||
fn request() -> ArtifactInventoryRequest {
|
||||
ArtifactInventoryRequest {
|
||||
protocol_version: VERSION,
|
||||
invocation_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
disk: "disk".into(),
|
||||
expected_disk_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
bucket: "artifacts".into(),
|
||||
bucket_incarnation_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
attempt_prefix: "outputs/tenant/run/attempt/".into(),
|
||||
timeout_millis: 30_000,
|
||||
phase: 0,
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn artifact_inventory_stream_rejects_scope_sequence_and_truncation() {
|
||||
let request = request();
|
||||
let frame = ArtifactInventoryFrame {
|
||||
sequence: 0,
|
||||
payload: Some(Payload::Complete(ArtifactInventoryComplete {
|
||||
disk_id: request.expected_disk_id.clone(),
|
||||
entries: 0,
|
||||
})),
|
||||
response_proof: vec![7; 32].into(),
|
||||
};
|
||||
let mut state = StreamState::new();
|
||||
assert!(state.finish().is_err(), "EOF alone is incomplete");
|
||||
state.accept(&request, &frame, |_, _| true).expect("A03I2_SIGNED_COMPLETION");
|
||||
state.finish().expect("authenticated terminal and EOF");
|
||||
assert!(state.accept(&request, &frame, |_, _| true).is_err(), "no data after Complete");
|
||||
let mut wrong = frame.clone();
|
||||
wrong.sequence = 1;
|
||||
assert!(
|
||||
StreamState::new().accept(&request, &wrong, |_, _| true).is_err(),
|
||||
"sequence starts at zero"
|
||||
);
|
||||
wrong = frame.clone();
|
||||
wrong.payload = Some(Payload::Verified(ArtifactInventoryVerified {
|
||||
disk_id: request.expected_disk_id.clone(),
|
||||
}));
|
||||
assert!(
|
||||
StreamState::new().accept(&request, &wrong, |_, _| true).is_err(),
|
||||
"verify is not scan completion"
|
||||
);
|
||||
let canonical = canonical_frame(&request, &frame).expect("canonical baseline");
|
||||
let mut changed = request.clone();
|
||||
changed.invocation_id = Uuid::new_v4().as_bytes().to_vec().into();
|
||||
assert_ne!(canonical, canonical_frame(&changed, &frame).expect("invocation"));
|
||||
changed = request.clone();
|
||||
changed.phase = 1;
|
||||
assert_ne!(canonical, canonical_frame(&changed, &frame).expect("phase"));
|
||||
}
|
||||
#[test]
|
||||
fn artifact_inventory_encoded_limits_and_full_request_binding() {
|
||||
use crate::proto_gen::node_service::{ArtifactInventoryEntries, ArtifactInventoryVersion};
|
||||
let baseline = request();
|
||||
let body = canonical_request(&baseline).expect("canonical request");
|
||||
let changes: Vec<fn(&mut ArtifactInventoryRequest)> = vec![
|
||||
|r| r.protocol_version += 1,
|
||||
|r| r.invocation_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
|r| r.disk.push('x'),
|
||||
|r| r.expected_disk_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
|r| r.bucket.push('x'),
|
||||
|r| r.bucket_incarnation_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
|r| r.attempt_prefix.push_str("other/"),
|
||||
|r| r.timeout_millis += 1,
|
||||
|r| r.phase = 1,
|
||||
];
|
||||
for change in changes {
|
||||
let mut changed = baseline.clone();
|
||||
change(&mut changed);
|
||||
assert_ne!(body, canonical_request(&changed).expect("bound field"));
|
||||
}
|
||||
let entry = ArtifactInventoryVersion {
|
||||
key: format!("{}object", baseline.attempt_prefix),
|
||||
version_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
delete_marker: false,
|
||||
};
|
||||
let frame = ArtifactInventoryFrame {
|
||||
sequence: 0,
|
||||
payload: Some(Payload::Entries(ArtifactInventoryEntries { versions: vec![entry] })),
|
||||
response_proof: vec![1; 32].into(),
|
||||
};
|
||||
let canonical = canonical_frame(&baseline, &frame).expect("baseline frame");
|
||||
for change in [0, 1, 2] {
|
||||
let mut changed = frame.clone();
|
||||
if let Some(Payload::Entries(entries)) = changed.payload.as_mut() {
|
||||
match change {
|
||||
0 => entries.versions[0].key.push('x'),
|
||||
1 => entries.versions[0].version_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
_ => entries.versions[0].delete_marker = true,
|
||||
}
|
||||
}
|
||||
assert_ne!(canonical, canonical_frame(&baseline, &changed).expect("all version fields bound"));
|
||||
}
|
||||
assert!(StreamState::new().accept(&baseline, &frame, |_, _| false).is_err(), "proof is mandatory");
|
||||
let mut oversized = frame.clone();
|
||||
if let Some(Payload::Entries(entries)) = oversized.payload.as_mut() {
|
||||
entries.versions[0].key = "x".repeat(MAX_FRAME_BYTES);
|
||||
}
|
||||
assert!(
|
||||
StreamState::new()
|
||||
.accept(&baseline, &oversized, |_, _| panic!("encoded limit checked before canonical copy"))
|
||||
.is_err()
|
||||
);
|
||||
let mut wrong = frame.clone();
|
||||
if let Some(Payload::Entries(entries)) = wrong.payload.as_mut() {
|
||||
entries.versions[0].version_id = vec![0; 16].into();
|
||||
}
|
||||
assert!(StreamState::new().accept(&baseline, &wrong, |_, _| true).is_err());
|
||||
let mut state = StreamState::new();
|
||||
state.accept(&baseline, &frame, |_, _| true).expect("one actual observation");
|
||||
assert!(state.finish().is_err(), "partial entries plus clean EOF is incomplete");
|
||||
let mut complete = ArtifactInventoryFrame {
|
||||
sequence: 1,
|
||||
payload: Some(Payload::Complete(ArtifactInventoryComplete {
|
||||
disk_id: baseline.expected_disk_id.clone(),
|
||||
entries: 0,
|
||||
})),
|
||||
response_proof: vec![1; 32].into(),
|
||||
};
|
||||
assert!(state.accept(&baseline, &complete, |_, _| true).is_err(), "terminal total must match");
|
||||
if let Some(Payload::Complete(value)) = complete.payload.as_mut() {
|
||||
value.entries = 1;
|
||||
}
|
||||
state.accept(&baseline, &complete, |_, _| true).expect("exact total");
|
||||
state.finish().expect("complete");
|
||||
let mut verify = baseline;
|
||||
verify.phase = 1;
|
||||
let terminal = ArtifactInventoryFrame {
|
||||
sequence: 0,
|
||||
payload: Some(Payload::Verified(ArtifactInventoryVerified {
|
||||
disk_id: verify.expected_disk_id.clone(),
|
||||
})),
|
||||
response_proof: vec![1; 32].into(),
|
||||
};
|
||||
StreamState::new()
|
||||
.accept(&verify, &terminal, |_, _| true)
|
||||
.expect("distinct read-only identity proof");
|
||||
}
|
||||
}
|
||||
@@ -434,7 +434,8 @@ pub struct RenameDataRequest {
|
||||
/// rename linearization point.
|
||||
#[prost(bytes = "bytes", tag = "8")]
|
||||
pub scanner_publication_lease_token: ::prost::bytes::Bytes,
|
||||
/// Required by RenameDataAtIncarnation; never accepted by legacy RenameData.
|
||||
/// Required by RenameDataAtIncarnation and ArtifactRenameData; never accepted
|
||||
/// by legacy RenameData. ArtifactRenameData also requires an empty scanner token.
|
||||
#[prost(bytes = "bytes", tag = "9")]
|
||||
pub bucket_incarnation_id: ::prost::bytes::Bytes,
|
||||
}
|
||||
@@ -449,6 +450,116 @@ pub struct RenameDataResponse {
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub rename_data_resp_bin: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Repair publication binds the required receiving physical disk separately from PUT.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ArtifactRepairRenameDataRequest {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub rename: ::core::option::Option<RenameDataRequest>,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub expected_disk_id: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ArtifactRepairWriteMetadataRequest {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub metadata: ::core::option::Option<WriteMetadataRequest>,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub expected_disk_id: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ArtifactInventoryRequest {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub protocol_version: u32,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub invocation_id: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "3")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub expected_disk_id: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "5")]
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "6")]
|
||||
pub bucket_incarnation_id: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "7")]
|
||||
pub attempt_prefix: ::prost::alloc::string::String,
|
||||
#[prost(uint64, tag = "8")]
|
||||
pub timeout_millis: u64,
|
||||
#[prost(enumeration = "ArtifactInventoryPhase", tag = "9")]
|
||||
pub phase: i32,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ArtifactInventoryVersion {
|
||||
#[prost(string, tag = "1")]
|
||||
pub key: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub version_id: ::prost::bytes::Bytes,
|
||||
#[prost(bool, tag = "3")]
|
||||
pub delete_marker: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct ArtifactInventoryEntries {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub versions: ::prost::alloc::vec::Vec<ArtifactInventoryVersion>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ArtifactInventoryComplete {
|
||||
#[prost(bytes = "bytes", tag = "1")]
|
||||
pub disk_id: ::prost::bytes::Bytes,
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub entries: u64,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ArtifactInventoryVerified {
|
||||
#[prost(bytes = "bytes", tag = "1")]
|
||||
pub disk_id: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct ArtifactInventoryFrame {
|
||||
#[prost(uint64, tag = "1")]
|
||||
pub sequence: u64,
|
||||
#[prost(bytes = "bytes", tag = "5")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
#[prost(oneof = "artifact_inventory_frame::Payload", tags = "2, 3, 4")]
|
||||
pub payload: ::core::option::Option<artifact_inventory_frame::Payload>,
|
||||
}
|
||||
/// Nested message and enum types in `ArtifactInventoryFrame`.
|
||||
pub mod artifact_inventory_frame {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Payload {
|
||||
#[prost(message, tag = "2")]
|
||||
Entries(super::ArtifactInventoryEntries),
|
||||
#[prost(message, tag = "3")]
|
||||
Complete(super::ArtifactInventoryComplete),
|
||||
#[prost(message, tag = "4")]
|
||||
Verified(super::ArtifactInventoryVerified),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SealArtifactAttemptPutsRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub bucket_incarnation_id: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "4")]
|
||||
pub attempt_prefix: ::prost::alloc::string::String,
|
||||
/// Required non-nil UUID of the physical disk admitted by the caller.
|
||||
#[prost(bytes = "bytes", tag = "5")]
|
||||
pub expected_disk_id: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SealArtifactAttemptPutsResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
/// Authenticates the complete canonical request and every response field below.
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
/// Required actual physical disk UUID; must match request.expected_disk_id.
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub disk_id: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct MakeVolumesRequest {
|
||||
/// indicate which one in the disks
|
||||
@@ -1831,6 +1942,34 @@ impl CompareAndUpdateFileOutcome {
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Receiver derives the managed prefix from authoritative bucket metadata.
|
||||
/// One invocation, no cursor or persistent scan session. Verify is never an inventory completion.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
|
||||
#[repr(i32)]
|
||||
pub enum ArtifactInventoryPhase {
|
||||
ArtifactInventoryScan = 0,
|
||||
ArtifactInventoryVerify = 1,
|
||||
}
|
||||
impl ArtifactInventoryPhase {
|
||||
/// String value of the enum field names used in the ProtoBuf definition.
|
||||
///
|
||||
/// The values are not transformed in any way and thus are considered stable
|
||||
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
|
||||
pub fn as_str_name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::ArtifactInventoryScan => "ARTIFACT_INVENTORY_SCAN",
|
||||
Self::ArtifactInventoryVerify => "ARTIFACT_INVENTORY_VERIFY",
|
||||
}
|
||||
}
|
||||
/// Creates an enum from field names used in the ProtoBuf definition.
|
||||
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
|
||||
match value {
|
||||
"ARTIFACT_INVENTORY_SCAN" => Some(Self::ArtifactInventoryScan),
|
||||
"ARTIFACT_INVENTORY_VERIFY" => Some(Self::ArtifactInventoryVerify),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
|
||||
#[repr(i32)]
|
||||
pub enum TierMutationPeerState {
|
||||
@@ -2453,6 +2592,96 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameDataAtIncarnation"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn artifact_rename_data(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ArtifactRenameData");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ArtifactRenameData"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn artifact_repair_rename_data(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ArtifactRepairRenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ArtifactRepairRenameData");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ArtifactRepairRenameData"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn artifact_repair_write_metadata(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ArtifactRepairWriteMetadataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::WriteMetadataResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ArtifactRepairWriteMetadata");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ArtifactRepairWriteMetadata"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn inventory_artifact_attempt(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ArtifactInventoryRequest>,
|
||||
) -> std::result::Result<tonic::Response<tonic::codec::Streaming<super::ArtifactInventoryFrame>>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/InventoryArtifactAttempt");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "InventoryArtifactAttempt"));
|
||||
self.inner.server_streaming(req, path, codec).await
|
||||
}
|
||||
pub async fn seal_artifact_attempt_repairs(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SealArtifactAttemptPutsRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SealArtifactAttemptPutsResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/SealArtifactAttemptRepairs");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "SealArtifactAttemptRepairs"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn seal_artifact_attempt_puts(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SealArtifactAttemptPutsRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SealArtifactAttemptPutsResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/SealArtifactAttemptPuts");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "SealArtifactAttemptPuts"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn rename_data_with_quota_fence(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::RenameDataRequest>,
|
||||
@@ -3716,6 +3945,34 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
|
||||
async fn artifact_rename_data(
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
|
||||
async fn artifact_repair_rename_data(
|
||||
&self,
|
||||
request: tonic::Request<super::ArtifactRepairRenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
|
||||
async fn artifact_repair_write_metadata(
|
||||
&self,
|
||||
request: tonic::Request<super::ArtifactRepairWriteMetadataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::WriteMetadataResponse>, tonic::Status>;
|
||||
/// Server streaming response type for the InventoryArtifactAttempt method.
|
||||
type InventoryArtifactAttemptStream: tonic::codegen::tokio_stream::Stream<Item = std::result::Result<super::ArtifactInventoryFrame, tonic::Status>>
|
||||
+ std::marker::Send
|
||||
+ 'static;
|
||||
async fn inventory_artifact_attempt(
|
||||
&self,
|
||||
request: tonic::Request<super::ArtifactInventoryRequest>,
|
||||
) -> std::result::Result<tonic::Response<Self::InventoryArtifactAttemptStream>, tonic::Status>;
|
||||
async fn seal_artifact_attempt_repairs(
|
||||
&self,
|
||||
request: tonic::Request<super::SealArtifactAttemptPutsRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SealArtifactAttemptPutsResponse>, tonic::Status>;
|
||||
async fn seal_artifact_attempt_puts(
|
||||
&self,
|
||||
request: tonic::Request<super::SealArtifactAttemptPutsRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SealArtifactAttemptPutsResponse>, tonic::Status>;
|
||||
async fn rename_data_with_quota_fence(
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataRequest>,
|
||||
@@ -4984,6 +5241,177 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ArtifactRenameData" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ArtifactRenameDataSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::RenameDataRequest> for ArtifactRenameDataSvc<T> {
|
||||
type Response = super::RenameDataResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::RenameDataRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::artifact_rename_data(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ArtifactRenameDataSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ArtifactRepairRenameData" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ArtifactRepairRenameDataSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::ArtifactRepairRenameDataRequest> for ArtifactRepairRenameDataSvc<T> {
|
||||
type Response = super::RenameDataResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::ArtifactRepairRenameDataRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::artifact_repair_rename_data(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ArtifactRepairRenameDataSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ArtifactRepairWriteMetadata" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ArtifactRepairWriteMetadataSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::ArtifactRepairWriteMetadataRequest>
|
||||
for ArtifactRepairWriteMetadataSvc<T>
|
||||
{
|
||||
type Response = super::WriteMetadataResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::ArtifactRepairWriteMetadataRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::artifact_repair_write_metadata(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ArtifactRepairWriteMetadataSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/InventoryArtifactAttempt" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct InventoryArtifactAttemptSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::ServerStreamingService<super::ArtifactInventoryRequest> for InventoryArtifactAttemptSvc<T> {
|
||||
type Response = super::ArtifactInventoryFrame;
|
||||
type ResponseStream = T::InventoryArtifactAttemptStream;
|
||||
type Future = BoxFuture<tonic::Response<Self::ResponseStream>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::ArtifactInventoryRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::inventory_artifact_attempt(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = InventoryArtifactAttemptSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.server_streaming(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/SealArtifactAttemptRepairs" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct SealArtifactAttemptRepairsSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::SealArtifactAttemptPutsRequest> for SealArtifactAttemptRepairsSvc<T> {
|
||||
type Response = super::SealArtifactAttemptPutsResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::SealArtifactAttemptPutsRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::seal_artifact_attempt_repairs(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = SealArtifactAttemptRepairsSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/SealArtifactAttemptPuts" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct SealArtifactAttemptPutsSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::SealArtifactAttemptPutsRequest> for SealArtifactAttemptPutsSvc<T> {
|
||||
type Response = super::SealArtifactAttemptPutsResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::SealArtifactAttemptPutsRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::seal_artifact_attempt_puts(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = SealArtifactAttemptPutsSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameDataWithQuotaFence" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameDataWithQuotaFenceSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod artifact_inventory;
|
||||
pub mod compat_manifest;
|
||||
// SAFETY: `generated` is prost/tonic-generated protocol code. The allowance is
|
||||
// scoped to that module so generated internals do not relax lints elsewhere.
|
||||
@@ -1084,6 +1085,119 @@ pub fn canonical_rename_data_request_body(
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
/// Artifact publication has its own mandatory body-bound RPC domain. The
|
||||
/// receiver validates the incarnation and rejects scanner tokens before writing.
|
||||
pub fn canonical_artifact_rename_data_request_body(
|
||||
request: &proto_gen::node_service::RenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-artifact-rename-data-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
body.push_bytes(&request.scanner_publication_lease_token)?;
|
||||
body.push_bytes(&request.bucket_incarnation_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
/// Repair operations always bind wrapper presence, both compatibility payloads,
|
||||
/// incarnation and receiving physical identity in independent operation domains.
|
||||
pub fn canonical_artifact_repair_rename_data_request_body(
|
||||
request: &proto_gen::node_service::ArtifactRepairRenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-artifact-repair-rename-data-request-v1\0");
|
||||
body.push_bool(request.rename.is_some());
|
||||
if let Some(rename) = &request.rename {
|
||||
body.push_str(&rename.disk)?;
|
||||
body.push_str(&rename.src_volume)?;
|
||||
body.push_str(&rename.src_path)?;
|
||||
body.push_str(&rename.file_info)?;
|
||||
body.push_str(&rename.dst_volume)?;
|
||||
body.push_str(&rename.dst_path)?;
|
||||
body.push_bytes(&rename.file_info_bin)?;
|
||||
body.push_bytes(&rename.scanner_publication_lease_token)?;
|
||||
body.push_bytes(&rename.bucket_incarnation_id)?;
|
||||
}
|
||||
body.push_bytes(&request.expected_disk_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_artifact_repair_write_metadata_request_body(
|
||||
request: &proto_gen::node_service::ArtifactRepairWriteMetadataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-artifact-repair-write-metadata-request-v1\0");
|
||||
body.push_bool(request.metadata.is_some());
|
||||
if let Some(metadata) = &request.metadata {
|
||||
body.push_str(&metadata.disk)?;
|
||||
body.push_str(&metadata.volume)?;
|
||||
body.push_str(&metadata.path)?;
|
||||
body.push_str(&metadata.file_info)?;
|
||||
body.push_bytes(&metadata.file_info_bin)?;
|
||||
body.push_bytes(&metadata.bucket_incarnation_id)?;
|
||||
}
|
||||
body.push_bytes(&request.expected_disk_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_seal_artifact_attempt_puts_request_body(
|
||||
request: &proto_gen::node_service::SealArtifactAttemptPutsRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-seal-artifact-attempt-puts-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_bytes(&request.bucket_incarnation_id)?;
|
||||
body.push_str(&request.attempt_prefix)?;
|
||||
body.push_bytes(&request.expected_disk_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
/// The proof covers the exact request and actual disk identity, so another
|
||||
/// disk, incarnation or attempt cannot reuse a successful seal acknowledgement.
|
||||
pub fn canonical_seal_artifact_attempt_puts_response_body(
|
||||
request: &proto_gen::node_service::SealArtifactAttemptPutsRequest,
|
||||
response: &proto_gen::node_service::SealArtifactAttemptPutsResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-seal-artifact-attempt-puts-response-v1\0");
|
||||
body.push_bytes(&canonical_seal_artifact_attempt_puts_request_body(request)?)?;
|
||||
body.push_bool(response.success);
|
||||
body.push_bool(response.error.is_some());
|
||||
body.push_u32(response.error.as_ref().map_or(0, |error| error.code));
|
||||
body.push_str(response.error.as_ref().map_or("", |error| error.error_info.as_str()))?;
|
||||
body.push_bytes(&response.disk_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_seal_artifact_attempt_repairs_request_body(
|
||||
request: &proto_gen::node_service::SealArtifactAttemptPutsRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-seal-artifact-attempt-repairs-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_bytes(&request.bucket_incarnation_id)?;
|
||||
body.push_str(&request.attempt_prefix)?;
|
||||
body.push_bytes(&request.expected_disk_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
/// The proof covers the exact request and actual disk identity, so another
|
||||
/// disk, incarnation or attempt cannot reuse a successful seal acknowledgement.
|
||||
pub fn canonical_seal_artifact_attempt_repairs_response_body(
|
||||
request: &proto_gen::node_service::SealArtifactAttemptPutsRequest,
|
||||
response: &proto_gen::node_service::SealArtifactAttemptPutsResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-seal-artifact-attempt-repairs-response-v1\0");
|
||||
body.push_bytes(&canonical_seal_artifact_attempt_repairs_request_body(request)?)?;
|
||||
body.push_bool(response.success);
|
||||
body.push_bool(response.error.is_some());
|
||||
body.push_u32(response.error.as_ref().map_or(0, |error| error.code));
|
||||
body.push_str(response.error.as_ref().map_or("", |error| error.error_info.as_str()))?;
|
||||
body.push_bytes(&response.disk_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub const QUOTA_MUTATION_TOPOLOGY_HEADER: &str = "x-rustfs-quota-topology";
|
||||
|
||||
/// Binds a quota mutation to the coordinator's complete, ordered storage layout.
|
||||
@@ -1542,6 +1656,447 @@ mod disk_mutation_canonical_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_rename_data_canonical_body_binds_every_field() {
|
||||
let baseline = RenameDataRequest {
|
||||
disk: "disk-a".into(),
|
||||
src_volume: "tmp".into(),
|
||||
src_path: "staged".into(),
|
||||
dst_volume: "artifacts".into(),
|
||||
dst_path: "outputs/tenant/run/attempt/image".into(),
|
||||
file_info: "{}".into(),
|
||||
file_info_bin: vec![0x80].into(),
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut bodies = vec![canonical_artifact_rename_data_request_body(&baseline).expect("artifact request")];
|
||||
for mutate in [
|
||||
|r: &mut RenameDataRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut RenameDataRequest| r.src_volume = "other-tmp".into(),
|
||||
|r: &mut RenameDataRequest| r.src_path = "other-stage".into(),
|
||||
|r: &mut RenameDataRequest| r.dst_volume = "other-artifacts".into(),
|
||||
|r: &mut RenameDataRequest| r.dst_path = "other-attempt/image".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info = "[]".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = vec![0x81].into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = Vec::new().into(),
|
||||
|r: &mut RenameDataRequest| r.scanner_publication_lease_token = vec![2; 16].into(),
|
||||
|r: &mut RenameDataRequest| r.bucket_incarnation_id = vec![3; 16].into(),
|
||||
|r: &mut RenameDataRequest| r.bucket_incarnation_id = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_artifact_rename_data_request_body(&request).expect("changed artifact request"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
assert_ne!(bodies[0], canonical_rename_data_request_body(&baseline).expect("legacy request"));
|
||||
let left = RenameDataRequest {
|
||||
src_volume: "ab".into(),
|
||||
src_path: "c".into(),
|
||||
..baseline.clone()
|
||||
};
|
||||
let right = RenameDataRequest {
|
||||
src_volume: "a".into(),
|
||||
src_path: "bc".into(),
|
||||
..baseline
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_artifact_rename_data_request_body(&left).expect("left fields"),
|
||||
canonical_artifact_rename_data_request_body(&right).expect("right fields")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_repair_rename_canonical_body_binds_wrapper_and_every_inner_field() {
|
||||
use proto_gen::node_service::ArtifactRepairRenameDataRequest;
|
||||
let baseline = ArtifactRepairRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: "disk-a".into(),
|
||||
src_volume: "tmp".into(),
|
||||
src_path: "stage".into(),
|
||||
dst_volume: "artifacts".into(),
|
||||
dst_path: "outputs/tenant/run/attempt/image".into(),
|
||||
file_info: "{}".into(),
|
||||
file_info_bin: vec![0x80].into(),
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
..Default::default()
|
||||
}),
|
||||
expected_disk_id: vec![2; 16].into(),
|
||||
};
|
||||
let original = canonical_artifact_repair_rename_data_request_body(&baseline).expect("repair canonical");
|
||||
let mut bodies = vec![original.clone()];
|
||||
for mutate in [
|
||||
|r: &mut RenameDataRequest| r.disk = "other-disk".into(),
|
||||
|r: &mut RenameDataRequest| r.src_volume = "other-tmp".into(),
|
||||
|r: &mut RenameDataRequest| r.src_path = "other-stage".into(),
|
||||
|r: &mut RenameDataRequest| r.dst_volume = "other-bucket".into(),
|
||||
|r: &mut RenameDataRequest| r.dst_path = "other-object".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info = "[]".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = vec![0x81].into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = Vec::new().into(),
|
||||
|r: &mut RenameDataRequest| r.scanner_publication_lease_token = vec![3; 16].into(),
|
||||
|r: &mut RenameDataRequest| r.bucket_incarnation_id = vec![4; 16].into(),
|
||||
|r: &mut RenameDataRequest| r.bucket_incarnation_id = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(request.rename.as_mut().expect("inner rename"));
|
||||
bodies.push(canonical_artifact_repair_rename_data_request_body(&request).expect("mutated canonical"));
|
||||
}
|
||||
for mutate in [
|
||||
|r: &mut ArtifactRepairRenameDataRequest| r.rename = None,
|
||||
|r: &mut ArtifactRepairRenameDataRequest| r.rename = Some(RenameDataRequest::default()),
|
||||
|r: &mut ArtifactRepairRenameDataRequest| r.expected_disk_id = vec![5; 16].into(),
|
||||
|r: &mut ArtifactRepairRenameDataRequest| r.expected_disk_id = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_artifact_repair_rename_data_request_body(&request).expect("wrapper canonical"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let rename = baseline.rename.as_ref().expect("rename");
|
||||
assert_ne!(original, canonical_artifact_rename_data_request_body(rename).expect("PUT canonical"));
|
||||
assert_ne!(original, canonical_rename_data_request_body(rename).expect("legacy canonical"));
|
||||
let mut left = baseline.clone();
|
||||
let mut right = baseline;
|
||||
left.rename.as_mut().expect("left").src_volume = "ab".into();
|
||||
left.rename.as_mut().expect("left").src_path = "c".into();
|
||||
right.rename.as_mut().expect("right").src_volume = "a".into();
|
||||
right.rename.as_mut().expect("right").src_path = "bc".into();
|
||||
assert_ne!(
|
||||
canonical_artifact_repair_rename_data_request_body(&left).expect("left"),
|
||||
canonical_artifact_repair_rename_data_request_body(&right).expect("right")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_repair_metadata_canonical_body_binds_wrapper_and_every_inner_field() {
|
||||
use proto_gen::node_service::{ArtifactRepairWriteMetadataRequest, WriteMetadataRequest};
|
||||
let baseline = ArtifactRepairWriteMetadataRequest {
|
||||
metadata: Some(WriteMetadataRequest {
|
||||
disk: "disk-a".into(),
|
||||
volume: "artifacts".into(),
|
||||
path: "outputs/tenant/run/attempt/image".into(),
|
||||
file_info: "{}".into(),
|
||||
file_info_bin: vec![0x80].into(),
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
}),
|
||||
expected_disk_id: vec![2; 16].into(),
|
||||
};
|
||||
let original = canonical_artifact_repair_write_metadata_request_body(&baseline).expect("metadata canonical");
|
||||
let mut bodies = vec![original.clone()];
|
||||
for mutate in [
|
||||
|r: &mut WriteMetadataRequest| r.disk = "other-disk".into(),
|
||||
|r: &mut WriteMetadataRequest| r.volume = "other-bucket".into(),
|
||||
|r: &mut WriteMetadataRequest| r.path = "other-object".into(),
|
||||
|r: &mut WriteMetadataRequest| r.file_info = "[]".into(),
|
||||
|r: &mut WriteMetadataRequest| r.file_info_bin = vec![0x81].into(),
|
||||
|r: &mut WriteMetadataRequest| r.file_info_bin = Vec::new().into(),
|
||||
|r: &mut WriteMetadataRequest| r.bucket_incarnation_id = vec![4; 16].into(),
|
||||
|r: &mut WriteMetadataRequest| r.bucket_incarnation_id = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(request.metadata.as_mut().expect("inner metadata"));
|
||||
bodies.push(canonical_artifact_repair_write_metadata_request_body(&request).expect("mutated metadata"));
|
||||
}
|
||||
for mutate in [
|
||||
|r: &mut ArtifactRepairWriteMetadataRequest| r.metadata = None,
|
||||
|r: &mut ArtifactRepairWriteMetadataRequest| r.metadata = Some(WriteMetadataRequest::default()),
|
||||
|r: &mut ArtifactRepairWriteMetadataRequest| r.expected_disk_id = vec![5; 16].into(),
|
||||
|r: &mut ArtifactRepairWriteMetadataRequest| r.expected_disk_id = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_artifact_repair_write_metadata_request_body(&request).expect("wrapper canonical"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
assert_ne!(
|
||||
original,
|
||||
canonical_write_metadata_request_body(baseline.metadata.as_ref().expect("metadata")).expect("legacy canonical")
|
||||
);
|
||||
let mut left = baseline.clone();
|
||||
let mut right = baseline;
|
||||
left.metadata.as_mut().expect("left").volume = "ab".into();
|
||||
left.metadata.as_mut().expect("left").path = "c".into();
|
||||
right.metadata.as_mut().expect("right").volume = "a".into();
|
||||
right.metadata.as_mut().expect("right").path = "bc".into();
|
||||
assert_ne!(
|
||||
canonical_artifact_repair_write_metadata_request_body(&left).expect("left"),
|
||||
canonical_artifact_repair_write_metadata_request_body(&right).expect("right")
|
||||
);
|
||||
}
|
||||
|
||||
fn seal_request() -> proto_gen::node_service::SealArtifactAttemptPutsRequest {
|
||||
proto_gen::node_service::SealArtifactAttemptPutsRequest {
|
||||
disk: "disk-a".into(),
|
||||
bucket: "artifacts".into(),
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
attempt_prefix: "outputs/tenant/run/attempt".into(),
|
||||
expected_disk_id: vec![2; 16].into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_put_seal_canonical_body_binds_every_request_field_and_response_scope() {
|
||||
use proto_gen::node_service::{SealArtifactAttemptPutsRequest, SealArtifactAttemptPutsResponse};
|
||||
let baseline = seal_request();
|
||||
let response = SealArtifactAttemptPutsResponse {
|
||||
success: true,
|
||||
disk_id: baseline.expected_disk_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut requests = vec![canonical_seal_artifact_attempt_puts_request_body(&baseline).expect("seal request")];
|
||||
let mut responses =
|
||||
vec![canonical_seal_artifact_attempt_puts_response_body(&baseline, &response).expect("seal response")];
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket = "other-bucket".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = vec![3; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = Vec::new().into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.attempt_prefix = "outputs/tenant/run/other".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = vec![4; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
requests.push(canonical_seal_artifact_attempt_puts_request_body(&request).expect("changed seal request"));
|
||||
responses.push(
|
||||
canonical_seal_artifact_attempt_puts_response_body(&request, &response).expect("changed seal response scope"),
|
||||
);
|
||||
}
|
||||
assert_all_distinct(&requests);
|
||||
assert_all_distinct(&responses);
|
||||
assert_ne!(requests[0], responses[0], "request and response domains must differ");
|
||||
let left = SealArtifactAttemptPutsRequest {
|
||||
disk: "ab".into(),
|
||||
bucket: "c".into(),
|
||||
..baseline.clone()
|
||||
};
|
||||
let right = SealArtifactAttemptPutsRequest {
|
||||
disk: "a".into(),
|
||||
bucket: "bc".into(),
|
||||
..baseline
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_seal_artifact_attempt_puts_request_body(&left).expect("left fields"),
|
||||
canonical_seal_artifact_attempt_puts_request_body(&right).expect("right fields")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_put_seal_canonical_response_binds_every_semantic_field() {
|
||||
use proto_gen::node_service::{Error, SealArtifactAttemptPutsResponse};
|
||||
let request = seal_request();
|
||||
let baseline = SealArtifactAttemptPutsResponse {
|
||||
success: true,
|
||||
error: Some(Error {
|
||||
code: 1,
|
||||
error_info: "sealed".into(),
|
||||
}),
|
||||
disk_id: request.expected_disk_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut bodies = vec![canonical_seal_artifact_attempt_puts_response_body(&request, &baseline).expect("seal response")];
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.success = false,
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.error = None,
|
||||
|r: &mut SealArtifactAttemptPutsResponse| {
|
||||
r.error = Some(Error {
|
||||
code: 2,
|
||||
error_info: "sealed".into(),
|
||||
})
|
||||
},
|
||||
|r: &mut SealArtifactAttemptPutsResponse| {
|
||||
r.error = Some(Error {
|
||||
code: 1,
|
||||
error_info: "other error".into(),
|
||||
})
|
||||
},
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.disk_id = vec![3; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.disk_id = Vec::new().into(),
|
||||
] {
|
||||
let mut response = baseline.clone();
|
||||
mutate(&mut response);
|
||||
bodies.push(canonical_seal_artifact_attempt_puts_response_body(&request, &response).expect("changed seal response"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let absent = SealArtifactAttemptPutsResponse {
|
||||
error: None,
|
||||
..baseline.clone()
|
||||
};
|
||||
let empty = SealArtifactAttemptPutsResponse {
|
||||
error: Some(Error::default()),
|
||||
..absent.clone()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_seal_artifact_attempt_puts_response_body(&request, &absent).expect("absent error"),
|
||||
canonical_seal_artifact_attempt_puts_response_body(&request, &empty).expect("present empty error")
|
||||
);
|
||||
let left = SealArtifactAttemptPutsResponse {
|
||||
error: Some(Error {
|
||||
code: 1,
|
||||
error_info: "ab".into(),
|
||||
}),
|
||||
disk_id: b"c".to_vec().into(),
|
||||
..baseline.clone()
|
||||
};
|
||||
let right = SealArtifactAttemptPutsResponse {
|
||||
error: Some(Error {
|
||||
code: 1,
|
||||
error_info: "a".into(),
|
||||
}),
|
||||
disk_id: b"bc".to_vec().into(),
|
||||
..baseline.clone()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_seal_artifact_attempt_puts_response_body(&request, &left).expect("left response fields"),
|
||||
canonical_seal_artifact_attempt_puts_response_body(&request, &right).expect("right response fields")
|
||||
);
|
||||
let signed = SealArtifactAttemptPutsResponse {
|
||||
response_proof: vec![9; 32].into(),
|
||||
..baseline
|
||||
};
|
||||
assert_eq!(
|
||||
canonical_seal_artifact_attempt_puts_response_body(&request, &signed).expect("proof exclusion"),
|
||||
bodies[0],
|
||||
"proof must not authenticate itself"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_repair_seal_canonical_body_binds_every_request_field_and_response_scope() {
|
||||
use proto_gen::node_service::{SealArtifactAttemptPutsRequest, SealArtifactAttemptPutsResponse};
|
||||
let baseline = seal_request();
|
||||
let response = SealArtifactAttemptPutsResponse {
|
||||
success: true,
|
||||
disk_id: baseline.expected_disk_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_seal_artifact_attempt_repairs_request_body(&baseline).expect("repair request"),
|
||||
canonical_seal_artifact_attempt_puts_request_body(&baseline).expect("PUT request")
|
||||
);
|
||||
assert_ne!(
|
||||
canonical_seal_artifact_attempt_repairs_response_body(&baseline, &response).expect("repair response"),
|
||||
canonical_seal_artifact_attempt_puts_response_body(&baseline, &response).expect("PUT response")
|
||||
);
|
||||
let mut requests = vec![canonical_seal_artifact_attempt_repairs_request_body(&baseline).expect("seal request")];
|
||||
let mut responses =
|
||||
vec![canonical_seal_artifact_attempt_repairs_response_body(&baseline, &response).expect("seal response")];
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket = "other-bucket".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = vec![3; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = Vec::new().into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.attempt_prefix = "outputs/tenant/run/other".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = vec![4; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
requests.push(canonical_seal_artifact_attempt_repairs_request_body(&request).expect("changed seal request"));
|
||||
responses.push(
|
||||
canonical_seal_artifact_attempt_repairs_response_body(&request, &response).expect("changed seal response scope"),
|
||||
);
|
||||
}
|
||||
assert_all_distinct(&requests);
|
||||
assert_all_distinct(&responses);
|
||||
assert_ne!(requests[0], responses[0], "request and response domains must differ");
|
||||
let left = SealArtifactAttemptPutsRequest {
|
||||
disk: "ab".into(),
|
||||
bucket: "c".into(),
|
||||
..baseline.clone()
|
||||
};
|
||||
let right = SealArtifactAttemptPutsRequest {
|
||||
disk: "a".into(),
|
||||
bucket: "bc".into(),
|
||||
..baseline
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_seal_artifact_attempt_repairs_request_body(&left).expect("left fields"),
|
||||
canonical_seal_artifact_attempt_repairs_request_body(&right).expect("right fields")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_repair_seal_canonical_response_binds_every_semantic_field() {
|
||||
use proto_gen::node_service::{Error, SealArtifactAttemptPutsResponse};
|
||||
let request = seal_request();
|
||||
let baseline = SealArtifactAttemptPutsResponse {
|
||||
success: true,
|
||||
error: Some(Error {
|
||||
code: 1,
|
||||
error_info: "sealed".into(),
|
||||
}),
|
||||
disk_id: request.expected_disk_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut bodies = vec![canonical_seal_artifact_attempt_repairs_response_body(&request, &baseline).expect("seal response")];
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.success = false,
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.error = None,
|
||||
|r: &mut SealArtifactAttemptPutsResponse| {
|
||||
r.error = Some(Error {
|
||||
code: 2,
|
||||
error_info: "sealed".into(),
|
||||
})
|
||||
},
|
||||
|r: &mut SealArtifactAttemptPutsResponse| {
|
||||
r.error = Some(Error {
|
||||
code: 1,
|
||||
error_info: "other error".into(),
|
||||
})
|
||||
},
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.disk_id = vec![3; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.disk_id = Vec::new().into(),
|
||||
] {
|
||||
let mut response = baseline.clone();
|
||||
mutate(&mut response);
|
||||
bodies
|
||||
.push(canonical_seal_artifact_attempt_repairs_response_body(&request, &response).expect("changed seal response"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let absent = SealArtifactAttemptPutsResponse {
|
||||
error: None,
|
||||
..baseline.clone()
|
||||
};
|
||||
let empty = SealArtifactAttemptPutsResponse {
|
||||
error: Some(Error::default()),
|
||||
..absent.clone()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_seal_artifact_attempt_repairs_response_body(&request, &absent).expect("absent error"),
|
||||
canonical_seal_artifact_attempt_repairs_response_body(&request, &empty).expect("present empty error")
|
||||
);
|
||||
let left = SealArtifactAttemptPutsResponse {
|
||||
error: Some(Error {
|
||||
code: 1,
|
||||
error_info: "ab".into(),
|
||||
}),
|
||||
disk_id: b"c".to_vec().into(),
|
||||
..baseline.clone()
|
||||
};
|
||||
let right = SealArtifactAttemptPutsResponse {
|
||||
error: Some(Error {
|
||||
code: 1,
|
||||
error_info: "a".into(),
|
||||
}),
|
||||
disk_id: b"bc".to_vec().into(),
|
||||
..baseline.clone()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_seal_artifact_attempt_repairs_response_body(&request, &left).expect("left response fields"),
|
||||
canonical_seal_artifact_attempt_repairs_response_body(&request, &right).expect("right response fields")
|
||||
);
|
||||
let signed = SealArtifactAttemptPutsResponse {
|
||||
response_proof: vec![9; 32].into(),
|
||||
..baseline
|
||||
};
|
||||
assert_eq!(
|
||||
canonical_seal_artifact_attempt_repairs_response_body(&request, &signed).expect("proof exclusion"),
|
||||
bodies[0],
|
||||
"proof must not authenticate itself"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_canonical_body_binds_every_field() {
|
||||
let baseline = DeleteVersionRequest {
|
||||
|
||||
@@ -328,7 +328,8 @@ message RenameDataRequest {
|
||||
// legacy rename request body; a non-empty token is checked at the target's
|
||||
// rename linearization point.
|
||||
bytes scanner_publication_lease_token = 8;
|
||||
// Required by RenameDataAtIncarnation; never accepted by legacy RenameData.
|
||||
// Required by RenameDataAtIncarnation and ArtifactRenameData; never accepted
|
||||
// by legacy RenameData. ArtifactRenameData also requires an empty scanner token.
|
||||
bytes bucket_incarnation_id = 9;
|
||||
}
|
||||
|
||||
@@ -339,6 +340,70 @@ message RenameDataResponse {
|
||||
bytes rename_data_resp_bin = 4;
|
||||
}
|
||||
|
||||
// Repair publication binds the required receiving physical disk separately from PUT.
|
||||
message ArtifactRepairRenameDataRequest {
|
||||
RenameDataRequest rename = 1;
|
||||
bytes expected_disk_id = 2;
|
||||
}
|
||||
|
||||
message ArtifactRepairWriteMetadataRequest {
|
||||
WriteMetadataRequest metadata = 1;
|
||||
bytes expected_disk_id = 2;
|
||||
}
|
||||
|
||||
// Receiver derives the managed prefix from authoritative bucket metadata.
|
||||
// One invocation, no cursor or persistent scan session. Verify is never an inventory completion.
|
||||
enum ArtifactInventoryPhase {
|
||||
ARTIFACT_INVENTORY_SCAN = 0;
|
||||
ARTIFACT_INVENTORY_VERIFY = 1;
|
||||
}
|
||||
message ArtifactInventoryRequest {
|
||||
uint32 protocol_version = 1;
|
||||
bytes invocation_id = 2;
|
||||
string disk = 3;
|
||||
bytes expected_disk_id = 4;
|
||||
string bucket = 5;
|
||||
bytes bucket_incarnation_id = 6;
|
||||
string attempt_prefix = 7;
|
||||
uint64 timeout_millis = 8;
|
||||
ArtifactInventoryPhase phase = 9;
|
||||
}
|
||||
message ArtifactInventoryVersion {
|
||||
string key = 1;
|
||||
bytes version_id = 2;
|
||||
bool delete_marker = 3;
|
||||
}
|
||||
message ArtifactInventoryEntries { repeated ArtifactInventoryVersion versions = 1; }
|
||||
message ArtifactInventoryComplete { bytes disk_id = 1; uint64 entries = 2; }
|
||||
message ArtifactInventoryVerified { bytes disk_id = 1; }
|
||||
message ArtifactInventoryFrame {
|
||||
uint64 sequence = 1;
|
||||
oneof payload {
|
||||
ArtifactInventoryEntries entries = 2;
|
||||
ArtifactInventoryComplete complete = 3;
|
||||
ArtifactInventoryVerified verified = 4;
|
||||
}
|
||||
bytes response_proof = 5;
|
||||
}
|
||||
|
||||
message SealArtifactAttemptPutsRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
bytes bucket_incarnation_id = 3;
|
||||
string attempt_prefix = 4;
|
||||
// Required non-nil UUID of the physical disk admitted by the caller.
|
||||
bytes expected_disk_id = 5;
|
||||
}
|
||||
|
||||
message SealArtifactAttemptPutsResponse {
|
||||
bool success = 1;
|
||||
optional Error error = 2;
|
||||
// Authenticates the complete canonical request and every response field below.
|
||||
bytes response_proof = 3;
|
||||
// Required actual physical disk UUID; must match request.expected_disk_id.
|
||||
bytes disk_id = 4;
|
||||
}
|
||||
|
||||
message MakeVolumesRequest {
|
||||
string disk = 1; // indicate which one in the disks
|
||||
repeated string volumes = 2;
|
||||
@@ -1286,6 +1351,12 @@ service NodeService {
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc ArtifactRenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc ArtifactRepairRenameData(ArtifactRepairRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc ArtifactRepairWriteMetadata(ArtifactRepairWriteMetadataRequest) returns (WriteMetadataResponse) {}; // auth-policy: body-bound
|
||||
rpc InventoryArtifactAttempt(ArtifactInventoryRequest) returns (stream ArtifactInventoryFrame) {}; // auth-policy: body-bound
|
||||
rpc SealArtifactAttemptRepairs(SealArtifactAttemptPutsRequest) returns (SealArtifactAttemptPutsResponse) {}; // auth-policy: body-bound
|
||||
rpc SealArtifactAttemptPuts(SealArtifactAttemptPutsRequest) returns (SealArtifactAttemptPutsResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataWithQuotaFence(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc MakeVolumes(MakeVolumesRequest) returns (MakeVolumesResponse) {}; // auth-policy: body-bound
|
||||
rpc MakeVolume(MakeVolumeRequest) returns (MakeVolumeResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -43,6 +43,9 @@ serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
rustfs-utils = { workspace = true, default-features = false, features = ["path"] }
|
||||
|
||||
[dev-dependencies]
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
@@ -24,6 +24,9 @@ pub const BUCKET_LIFECYCLE_LOCK_OBJECT: &str = "";
|
||||
/// Options for creating a new bucket.
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct MakeBucketOptions {
|
||||
/// Immutable output namespace, set only when a dedicated artifact bucket is created.
|
||||
#[serde(default)]
|
||||
pub artifact_base_prefix: Option<String>,
|
||||
/// Enable object lock for the bucket.
|
||||
pub lock_enabled: bool,
|
||||
/// Enable versioning for the bucket.
|
||||
@@ -36,6 +39,31 @@ pub struct MakeBucketOptions {
|
||||
pub no_lock: bool,
|
||||
}
|
||||
|
||||
// The shortest object suffix has a nonempty hex tenant (2 bytes), two
|
||||
// hyphenated UUIDs (36 each), three separators, and a one-byte object name.
|
||||
const MIN_ARTIFACT_OBJECT_SUFFIX_BYTES: usize = 2 + 2 * 36 + 3 + 1;
|
||||
|
||||
/// Validate a relative artifact namespace with room for at least one object.
|
||||
/// Longer tenant IDs and object names still need full-key validation.
|
||||
pub fn valid_artifact_base_prefix(prefix: &str) -> bool {
|
||||
prefix.strip_suffix('/').is_some_and(|base| {
|
||||
prefix.len() <= 1024 - MIN_ARTIFACT_OBJECT_SUFFIX_BYTES
|
||||
&& prefix
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || b"/._-".contains(&byte))
|
||||
&& base.split('/').all(|part| {
|
||||
if part.is_empty() || part.len() > 255 || matches!(part, "." | "..") {
|
||||
return false;
|
||||
}
|
||||
#[cfg(windows)]
|
||||
if rustfs_utils::check_windows_file_name(part).is_err() {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Operation to perform on a bucket during site replication delete.
|
||||
#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum SRBucketDeleteOp {
|
||||
@@ -104,6 +132,79 @@ pub trait BucketOperations: Send + Sync + Debug {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn artifact_creation_option_survives_serialization() {
|
||||
let mut request = serde_json::to_value(MakeBucketOptions::default()).expect("encode creation options");
|
||||
request["artifact_base_prefix"] = "artifacts/".into();
|
||||
let options: MakeBucketOptions = serde_json::from_value(request).expect("decode artifact creation");
|
||||
assert_eq!(
|
||||
serde_json::to_value(options).expect("encode artifact creation")["artifact_base_prefix"],
|
||||
"artifacts/",
|
||||
"ARTIFACT_CREATION_OPTION"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_prefix_matches_runner_encoding() {
|
||||
for prefix in ["outputs/", "artifacts/v1._-/", "nested/", "nested/level/"] {
|
||||
assert!(valid_artifact_base_prefix(prefix), "ARTIFACT_PREFIX_VALID: {prefix}");
|
||||
}
|
||||
for prefix in [
|
||||
"",
|
||||
"outputs",
|
||||
"/",
|
||||
"//",
|
||||
"/nested/",
|
||||
"nested//",
|
||||
"nested//level/",
|
||||
"nested/level//",
|
||||
"../",
|
||||
"a/./",
|
||||
"a/../",
|
||||
"空/",
|
||||
"a b/",
|
||||
"a%2f/",
|
||||
] {
|
||||
assert!(!valid_artifact_base_prefix(prefix), "ARTIFACT_PREFIX_INVALID: {prefix}");
|
||||
}
|
||||
assert!(!valid_artifact_base_prefix(&format!("{}/", "a".repeat(1024))));
|
||||
let legacy = serde_json::json!({"lock_enabled":false,"versioning_enabled":false,"force_create":false,"created_at":null,"no_lock":false});
|
||||
let options: MakeBucketOptions = serde_json::from_value(legacy).expect("decode legacy creation options");
|
||||
assert_eq!(options.artifact_base_prefix, None, "ARTIFACT_LEGACY_CREATION");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_prefix_components_fit_on_disk() {
|
||||
assert!(valid_artifact_base_prefix(&format!("{}/", "a".repeat(255))));
|
||||
assert!(!valid_artifact_base_prefix(&format!("{}/", "a".repeat(256))));
|
||||
assert!(!valid_artifact_base_prefix(&format!("outputs/{}/nested/", "a".repeat(256))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_prefix_leaves_room_for_the_shortest_object() {
|
||||
for (last_segment_bytes, valid) in [(177, true), (178, false)] {
|
||||
let prefix = format!(
|
||||
"{}/{}/{}/{}/",
|
||||
"a".repeat(255),
|
||||
"b".repeat(255),
|
||||
"c".repeat(255),
|
||||
"d".repeat(last_segment_bytes)
|
||||
);
|
||||
assert_eq!(prefix.len(), if valid { 946 } else { 947 });
|
||||
assert_eq!(valid_artifact_base_prefix(&prefix), valid, "ARTIFACT_PREFIX_OBJECT_BUDGET");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_prefix_preserves_host_file_name_rules() {
|
||||
for prefix in ["NUL/", "con.txt/", "nested/COM1/", "bad./"] {
|
||||
assert_eq!(valid_artifact_base_prefix(prefix), !cfg!(windows), "ARTIFACT_PREFIX_HOST_NAME: {prefix}");
|
||||
}
|
||||
for prefix in ["normal/", "nul-name/", "v1.0/"] {
|
||||
assert!(valid_artifact_base_prefix(prefix), "ARTIFACT_PREFIX_HOST_VALID: {prefix}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_info_serializes_with_existing_fields() {
|
||||
let bucket = BucketInfo {
|
||||
|
||||
@@ -27,9 +27,10 @@ pub(crate) mod fixture {
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::api::object::{PutObjReader, SelectObjectSnapshot};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
|
||||
#[cfg(feature = "put-object-commit-barrier")]
|
||||
pub(crate) use rustfs_ecstore::api::set_disk as ecstore_set_disk;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks, init_local_disks_with_instance_ctx};
|
||||
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, ObjectIO};
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -35,8 +35,9 @@ use std::sync::{Arc, Once};
|
||||
#[cfg(feature = "put-object-commit-barrier")]
|
||||
use ecstore_test_compat::fixture::ecstore_set_disk;
|
||||
use ecstore_test_compat::fixture::{
|
||||
BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
|
||||
PoolEndpoints, PutObjReader, SelectObjectSnapshot, init_bucket_metadata_sys, init_local_disks,
|
||||
BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, MakeBucketOptions,
|
||||
ObjectIO as _, PoolEndpoints, PutObjReader, SelectObjectSnapshot, init_bucket_metadata_sys, init_local_disks,
|
||||
init_local_disks_with_instance_ctx,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -90,7 +91,7 @@ pub struct TestECStoreEnv {
|
||||
pub temp_root: PathBuf,
|
||||
/// The per-disk directories (`disk1`..`diskN`) under `temp_root`.
|
||||
pub disk_paths: Vec<PathBuf>,
|
||||
/// The store, bootstrapped exactly like the historical test setups:
|
||||
/// By default, the store is bootstrapped exactly like the historical test setups:
|
||||
/// `init_local_disks` + `ECStore::new` on `127.0.0.1:0` (random port keeps
|
||||
/// nextest's process-per-test parallelism safe).
|
||||
pub ecstore: Arc<ECStore>,
|
||||
@@ -155,6 +156,7 @@ pub struct TestECStoreEnvBuilder {
|
||||
prefix: String,
|
||||
base_dir: Option<PathBuf>,
|
||||
init_bucket_metadata: bool,
|
||||
instance_ctx: Option<Arc<InstanceContext>>,
|
||||
}
|
||||
|
||||
impl Default for TestECStoreEnvBuilder {
|
||||
@@ -164,6 +166,7 @@ impl Default for TestECStoreEnvBuilder {
|
||||
prefix: "rustfs_test_utils".to_string(),
|
||||
base_dir: None,
|
||||
init_bucket_metadata: true,
|
||||
instance_ctx: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -196,6 +199,12 @@ impl TestECStoreEnvBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Use a caller-owned context for local disks and store construction.
|
||||
pub fn instance_ctx(mut self, instance_ctx: Arc<InstanceContext>) -> Self {
|
||||
self.instance_ctx = Some(instance_ctx);
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the environment. Panics on any bootstrap failure — this is test
|
||||
/// scaffolding, and a broken environment must fail the test loudly.
|
||||
pub async fn build(self) -> TestECStoreEnv {
|
||||
@@ -237,14 +246,22 @@ impl TestECStoreEnvBuilder {
|
||||
};
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
|
||||
|
||||
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
|
||||
|
||||
// Port 0 keeps ECStore-backed integration binaries parallel-safe under
|
||||
// nextest: no fixed peer port is ever shared between test processes.
|
||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().expect("parse test addr");
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new())
|
||||
.await
|
||||
.expect("build test ECStore");
|
||||
let ecstore = if let Some(instance_ctx) = self.instance_ctx {
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("init instance local disks");
|
||||
ECStore::new_with_instance_ctx(server_addr, endpoint_pools.clone(), CancellationToken::new(), instance_ctx)
|
||||
.await
|
||||
.expect("build instance test ECStore")
|
||||
} else {
|
||||
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
|
||||
ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new())
|
||||
.await
|
||||
.expect("build test ECStore")
|
||||
};
|
||||
|
||||
// The production bootstrap only persists pool.bin from the elected
|
||||
// first cluster node. Test stores intentionally have no cluster
|
||||
@@ -278,3 +295,86 @@ impl TestECStoreEnvBuilder {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod context_tests {
|
||||
use super::{InstanceContext, TestECStoreEnv};
|
||||
use crate::ecstore_test_compat::fixture::{read_config, save_config};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_context_keeps_stores_isolated() {
|
||||
let root_a = tempfile::tempdir().expect("create context A disk root");
|
||||
let root_b = tempfile::tempdir().expect("create context B disk root");
|
||||
let ctx_a = Arc::new(InstanceContext::new());
|
||||
let ctx_b = Arc::new(InstanceContext::new());
|
||||
let env_a = TestECStoreEnv::builder()
|
||||
.base_dir(root_a.path())
|
||||
.init_bucket_metadata(false)
|
||||
.instance_ctx(ctx_a.clone())
|
||||
.build()
|
||||
.await;
|
||||
let env_b = TestECStoreEnv::builder()
|
||||
.base_dir(root_b.path())
|
||||
.init_bucket_metadata(false)
|
||||
.instance_ctx(ctx_b.clone())
|
||||
.build()
|
||||
.await;
|
||||
|
||||
assert!(env_a.ecstore.instance_endpoints().is_none());
|
||||
assert!(env_b.ecstore.instance_endpoints().is_none());
|
||||
ctx_a.set_endpoints(env_a.endpoint_pools.clone());
|
||||
let endpoints_a = env_a
|
||||
.ecstore
|
||||
.instance_endpoints()
|
||||
.expect("instance A sees its caller-owned context topology");
|
||||
assert_eq!(
|
||||
endpoints_a.0[0]
|
||||
.endpoints
|
||||
.into_ref()
|
||||
.first()
|
||||
.expect("context A first endpoint")
|
||||
.to_string(),
|
||||
env_a.disk_paths[0].to_string_lossy(),
|
||||
);
|
||||
assert!(
|
||||
env_b.ecstore.instance_endpoints().is_none(),
|
||||
"publishing context A must not publish context B"
|
||||
);
|
||||
ctx_b.set_endpoints(env_b.endpoint_pools.clone());
|
||||
let endpoints_b = env_b
|
||||
.ecstore
|
||||
.instance_endpoints()
|
||||
.expect("instance B sees its caller-owned context topology");
|
||||
assert_eq!(
|
||||
endpoints_b.0[0]
|
||||
.endpoints
|
||||
.into_ref()
|
||||
.first()
|
||||
.expect("context B first endpoint")
|
||||
.to_string(),
|
||||
env_b.disk_paths[0].to_string_lossy(),
|
||||
);
|
||||
assert_eq!(
|
||||
env_a.ecstore.instance_endpoints().expect("context A remains published").0[0]
|
||||
.endpoints
|
||||
.into_ref()
|
||||
.first()
|
||||
.expect("context A retained endpoint")
|
||||
.to_string(),
|
||||
env_a.disk_paths[0].to_string_lossy(),
|
||||
);
|
||||
|
||||
let path = "config/test-context-isolation.json";
|
||||
let bytes_a = br#"{"context":"a"}"#.to_vec();
|
||||
let bytes_b = br#"{"context":"b"}"#.to_vec();
|
||||
save_config(env_a.ecstore.clone(), path, bytes_a.clone())
|
||||
.await
|
||||
.expect("write context A config");
|
||||
save_config(env_b.ecstore.clone(), path, bytes_b.clone())
|
||||
.await
|
||||
.expect("write context B config");
|
||||
assert_eq!(read_config(env_a.ecstore.clone(), path).await.expect("read context A config"), bytes_a);
|
||||
assert_eq!(read_config(env_b.ecstore.clone(), path).await.expect("read context B config"), bytes_b);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +50,8 @@ Required headings and strings in these files are asserted by `scripts/check_arch
|
||||
| [scanner-per-bucket-convergence.md](scanner-per-bucket-convergence.md) | changing what makes a scanner cycle `Complete`, `Superseded`, or converged, adding or re-classifying a dirty-usage producer, deciding whether a background writeback may invalidate a cycle, changing scanner worker concurrency or throttling, or activating scoped dirty acknowledgements |
|
||||
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
|
||||
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
|
||||
| [plugin-delegation.md](plugin-delegation.md) | changing plugin delegation registration, revocation, broker identity, or the scope of attempt credentials |
|
||||
| [artifact-storage.md](artifact-storage.md) | changing the creation, persisted identity, or versioning of a bucket reserved for plugin outputs |
|
||||
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
|
||||
| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another |
|
||||
| [tier-stats-contract.md](tier-stats-contract.md) | changing what `GET /rustfs/admin/v3/tier-stats` returns, adding a tier accounting source, or wiring a metric to a remote tier request |
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user