mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-27 14:27:10 +00:00
Compare commits
228 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1f8d013b8d | |||
| ceddcdd878 | |||
| b8858d0c70 | |||
| 7728eeab1a | |||
| 4fd444852c | |||
| 13dc635d51 | |||
| 05f84793a3 | |||
| 2fa9a433ad | |||
| 08811ec243 | |||
| 0f89e9b8b8 | |||
| de8b465c41 | |||
| ead77d799f | |||
| 67db0f241f | |||
| a94f625bc8 | |||
| b7835f0f9a | |||
| 299609bb83 | |||
| 1800b90d06 | |||
| 2ccc8d8647 | |||
| 03f77844b7 | |||
| b44c76db52 | |||
| d0096af742 | |||
| 35e07b44f4 | |||
| 7d91176cea | |||
| 001d935569 | |||
| cc157adca8 | |||
| e0917fb617 | |||
| 6a30694765 | |||
| 991d647c78 | |||
| aafa85c6e5 | |||
| 01f5aa9c3a | |||
| e27bfb2006 | |||
| f095c6a476 | |||
| a223e74c83 | |||
| 1d1964215d | |||
| 5b23462652 | |||
| 02f6d87201 | |||
| 37a54d3ff2 | |||
| d1aaaee9bf | |||
| bebd5d0631 | |||
| 0de18a3577 | |||
| 04756b0a93 | |||
| 6fc62edcf7 | |||
| 68d1790b74 | |||
| 6fde372f5b | |||
| 503a852214 | |||
| ab7a8c400f | |||
| 85663e9647 | |||
| 61e4cfbc2a | |||
| 675722e5f8 | |||
| f6dfe86c58 | |||
| 3c9b9589f2 | |||
| a5ad283c8c | |||
| dcdd53ccf3 | |||
| bdd7efc96d | |||
| 9fbf5d0553 | |||
| e7bebb0fc5 | |||
| cacdaa65a0 | |||
| efe2255ccf | |||
| f4b694ed19 | |||
| aa95a32ea2 | |||
| 3f3e892d86 | |||
| 874d2babcc | |||
| 7c85783a08 | |||
| 541fbbe74a | |||
| 08ba7a7976 | |||
| 81f214c99a | |||
| dff05785de | |||
| cf2e3af4d9 | |||
| e0f0628bc1 | |||
| 6de3034e36 | |||
| ae259e127d | |||
| 8233e3fe4c | |||
| f88b7b6ba5 | |||
| 0307c9a7e2 | |||
| 6a8d0aba58 | |||
| ee6c2afe43 | |||
| 9ca8bc3ca6 | |||
| 3678703a98 | |||
| fe798c13e7 | |||
| 24b77349aa | |||
| 8c55396bfe | |||
| 5be8badb20 | |||
| 46daaadbb3 | |||
| f3b6c4df6b | |||
| 7183849227 | |||
| 942ce79c4e | |||
| fec1262f1a | |||
| 5eb46cb29d | |||
| 2cd904d0e7 | |||
| 61eec6da18 | |||
| 5b8cd2a5dc | |||
| d7824049b2 | |||
| d46e4c54a6 | |||
| 3ca0b5f35e | |||
| 00b5ddc0ad | |||
| ad4c2bc9b1 | |||
| ef5fe69d52 | |||
| ba6f37e58d | |||
| 31eb2e5a4a | |||
| 2f3654793d | |||
| ce09e7817c | |||
| 31dc8a0fea | |||
| d243a4a29e | |||
| 1048374cbd | |||
| de32dece41 | |||
| f79f318ec4 | |||
| 60fe68a04a | |||
| 498be1fa53 | |||
| b798d291d7 | |||
| 57c4d4c719 | |||
| 560b083278 | |||
| 36afe8bb57 | |||
| 57963574b7 | |||
| c56a597374 | |||
| 5354e6dccf | |||
| 7608c5af11 | |||
| c926e10de0 | |||
| 972dafccde | |||
| 5116bb2bbb | |||
| 08d6e4343d | |||
| 7d3a6acc57 | |||
| aabb26b519 | |||
| 163e6064da | |||
| c7a9b76543 | |||
| 7cc4ce5447 | |||
| 44fe49e5fa | |||
| dde2c50309 | |||
| d5a8f022bd | |||
| 150d349ade | |||
| 38056f3783 | |||
| 0b40e35e76 | |||
| 458b144ac7 | |||
| d58b80d54e | |||
| f8d0174fa7 | |||
| a57cc0d44c | |||
| 8006cb7eae | |||
| 8d40a995a3 | |||
| a451b53bcc | |||
| a6127c8fde | |||
| f346c13e5b | |||
| 2634e829ca | |||
| dc15444256 | |||
| ebb19907a9 | |||
| d29babf251 | |||
| d8a9522919 | |||
| 60dc84bf0e | |||
| 061e89ed2b | |||
| 3065142735 | |||
| c6a8e30f5a | |||
| f51b52d7e9 | |||
| c54160c22c | |||
| 1f67b65ab8 | |||
| a7806e65f0 | |||
| 6a3678997a | |||
| 64ea8b8e09 | |||
| 7773aac0fd | |||
| 8763f87ca7 | |||
| 9bbac59977 | |||
| 3bd57517ff | |||
| 8ee78c6891 | |||
| 21c594e625 | |||
| 748095280a | |||
| 960195458f | |||
| de58fdfc7d | |||
| 869d5ec24b | |||
| 19902438a6 | |||
| bd244d13f5 | |||
| 38395d35ec | |||
| 68064b5ad9 | |||
| c1da0c361b | |||
| cb7626c1f1 | |||
| 8186a7175c | |||
| c9e1446126 | |||
| 11d21ec6f4 | |||
| 974ab6f785 | |||
| c3f4cdf27c | |||
| e82d7c107a | |||
| 9d2d6a8e46 | |||
| ff2cbc3c47 | |||
| d65ef3e29a | |||
| 0a2f58c7a9 | |||
| 4d97501327 | |||
| 3b2af12c64 | |||
| c73ddde848 | |||
| ef72a37fd7 | |||
| e46cf750dd | |||
| fe955fa3ee | |||
| 64578c067e | |||
| 78715f0921 | |||
| 3284aafd73 | |||
| 9286442f87 | |||
| b8aad1c168 | |||
| 5fcb229760 | |||
| 94baef7462 | |||
| 60d90f085a | |||
| dfa5d50565 | |||
| a902886737 | |||
| 8e53b87421 | |||
| f7631e91b1 | |||
| f9bc4e921f | |||
| 781584307a | |||
| 59d641a5fe | |||
| c618b57870 | |||
| bbe1d54098 | |||
| 9e8ea4dcdb | |||
| 63cf8e2f2f | |||
| b7fe40f4b3 | |||
| 5b872b8e56 | |||
| ca5be1ef95 | |||
| 9373e8fa3e | |||
| cb32965522 | |||
| fb6cf49a67 | |||
| 8c97049e68 | |||
| 17d4fe27cc | |||
| fb0c7972b2 | |||
| eba7967755 | |||
| 451c1c31ad | |||
| 8780dd670e | |||
| f29bf2c902 | |||
| fd03436090 | |||
| 1f1e7b1c97 | |||
| 45a952609e | |||
| 323190a95a | |||
| 38cdda9292 | |||
| 85c84fa73d | |||
| 51402087c5 | |||
| a994984fe2 | |||
| 48eb0e402c |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=cf31ed27e4e6bf42da5bf81f248d9d2619ac8e8615186cdacb30bd63996d7607
|
||||
sha256-darwin=cf31ed27e4e6bf42da5bf81f248d9d2619ac8e8615186cdacb30bd63996d7607
|
||||
sha256-linux=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
|
||||
sha256-darwin=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
|
||||
|
||||
@@ -25,6 +25,8 @@ on:
|
||||
- "crates/ecstore/src/object_api/write_commit_context.rs"
|
||||
- "crates/ecstore/src/bucket/utils.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "crates/storage-api/src/bucket.rs"
|
||||
- "crates/storage-api/Cargo.toml"
|
||||
- "crates/utils/src/dunce.rs"
|
||||
- "crates/utils/src/os/**"
|
||||
- "crates/utils/Cargo.toml"
|
||||
@@ -48,6 +50,8 @@ on:
|
||||
- "crates/ecstore/src/object_api/write_commit_context.rs"
|
||||
- "crates/ecstore/src/bucket/utils.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "crates/storage-api/src/bucket.rs"
|
||||
- "crates/storage-api/Cargo.toml"
|
||||
- "crates/utils/src/dunce.rs"
|
||||
- "crates/utils/src/os/**"
|
||||
- "crates/utils/Cargo.toml"
|
||||
@@ -140,6 +144,17 @@ jobs:
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-ecstore --lib bucket::utils:: -- --nocapture"
|
||||
|
||||
- name: Test artifact prefix Windows file-name rules
|
||||
shell: pwsh
|
||||
run: |
|
||||
$testName = 'bucket::tests::artifact_prefix_preserves_host_file_name_rules'
|
||||
cargo test --locked -p rustfs-storage-api --lib $testName -- --exact --nocapture --color never 2>&1 |
|
||||
Tee-Object -Variable testOutput
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
if (($testOutput -join "`n") -notmatch 'test result: ok\. 1 passed; 0 failed; 0 ignored;') {
|
||||
throw 'Expected exactly one passing artifact prefix test on Windows'
|
||||
}
|
||||
|
||||
- name: Test Windows file-name rules
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
|
||||
|
||||
Generated
+2
@@ -10298,6 +10298,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10687,6 +10688,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
|
||||
@@ -200,8 +200,8 @@ python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run
|
||||
python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
# Distributed 4-node 4-disk lane uses the fault-test hooks server; the
|
||||
# scanner_heal_topology_ fault cases fail closed without it.
|
||||
# Distributed 4-node 4-disk lane includes physical-worker fault injection and
|
||||
# scanner_heal_topology_ cases; both require the fault-test hooks server.
|
||||
# Upgrade cases require RUSTFS_UPGRADE_SOURCE_BINARY and fail closed without it.
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, payload_for, s3_probe_client, unique_bucket, wait_for_ready,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier};
|
||||
use serde_json::json;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const PUBLIC_INTERNAL_ERROR: &str = "We encountered an internal error, please try again.";
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub(super) struct PhysicalEntry {
|
||||
identity: (u64, u64),
|
||||
changed: (i64, i64),
|
||||
modified: (i64, i64),
|
||||
bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
// Include directory identity/ctime as well as file bytes. Creating and removing
|
||||
// a temporary marker/rollback file cannot hide behind a restored final xl.meta.
|
||||
// Reads change atime, which is deliberately excluded.
|
||||
pub(super) fn physical_snapshot(dist: &DistCluster, bucket: &str) -> TestResult<BTreeMap<(usize, PathBuf), PhysicalEntry>> {
|
||||
let mut snapshot = BTreeMap::new();
|
||||
for (node, instance) in dist.cluster.nodes.iter().enumerate() {
|
||||
let root = Path::new(&instance.data_dirs[0]).join(bucket);
|
||||
for entry in walkdir::WalkDir::new(&root) {
|
||||
let entry = entry?;
|
||||
let metadata = std::fs::symlink_metadata(entry.path())?;
|
||||
assert!(!metadata.file_type().is_symlink(), "unexpected fixture symlink");
|
||||
snapshot.insert(
|
||||
(node, entry.path().strip_prefix(&root)?.to_path_buf()),
|
||||
PhysicalEntry {
|
||||
identity: (metadata.dev(), metadata.ino()),
|
||||
changed: (metadata.ctime(), metadata.ctime_nsec()),
|
||||
modified: (metadata.mtime(), metadata.mtime_nsec()),
|
||||
bytes: if metadata.is_dir() {
|
||||
None
|
||||
} else {
|
||||
Some(std::fs::read(entry.path())?)
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
struct Version {
|
||||
key: String,
|
||||
id: String,
|
||||
body: Vec<u8>,
|
||||
manifests: Vec<VersionShardCensus>,
|
||||
}
|
||||
|
||||
impl Version {
|
||||
async fn put(dist: &DistCluster, bucket: &str, key: &str, seed: &str) -> TestResult<Self> {
|
||||
let body = payload_for(seed, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let id = output.version_id().ok_or("fixture PUT omitted version ID")?.to_owned();
|
||||
let manifests = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, key, Some(&id)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
assert!(
|
||||
manifests
|
||||
.iter()
|
||||
.all(|manifest| manifest.is_complete() && !manifest.expected_part_numbers.is_empty())
|
||||
);
|
||||
Ok(Self {
|
||||
key: key.to_owned(),
|
||||
id,
|
||||
body,
|
||||
manifests,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn target(key: &str, version: Option<&str>) -> TestResult<ObjectIdentifier> {
|
||||
Ok(ObjectIdentifier::builder()
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.build()?)
|
||||
}
|
||||
|
||||
async fn assert_fixture(
|
||||
dist: &DistCluster,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
versions: &[Version],
|
||||
marker: (&str, &str),
|
||||
) -> TestResult {
|
||||
let expected: BTreeSet<_> = versions
|
||||
.iter()
|
||||
.map(|version| (version.key.clone(), version.id.clone()))
|
||||
.collect();
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node)?;
|
||||
let listed = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.list_object_versions().bucket(bucket).prefix(prefix).send(),
|
||||
)
|
||||
.await??;
|
||||
let actual: BTreeSet<_> = listed
|
||||
.versions()
|
||||
.iter()
|
||||
.map(|version| {
|
||||
(
|
||||
version.key().unwrap_or_default().to_owned(),
|
||||
version.version_id().unwrap_or_default().to_owned(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(listed.versions().len(), versions.len());
|
||||
assert_eq!(actual, expected, "delete changed exact data versions on node {node}");
|
||||
assert_eq!(listed.delete_markers().len(), 1);
|
||||
assert_eq!(listed.delete_markers()[0].key(), Some(marker.0));
|
||||
assert_eq!(listed.delete_markers()[0].version_id(), Some(marker.1));
|
||||
assert_eq!(listed.delete_markers()[0].is_latest(), Some(true));
|
||||
for version in versions {
|
||||
assert_exact_version(&client, bucket, &version.key, &version.id, &version.body).await?;
|
||||
let disk = Path::new(&dist.cluster.nodes[node].data_dirs[0]);
|
||||
assert!(
|
||||
census_object_version_on_disk(disk, bucket, &version.key, Some(&version.id))?
|
||||
.matches_manifest(&version.manifests[node])
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_denied(dist: &DistCluster, node: usize, bucket: &str, key: &str, version: Option<&str>) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
// One attempt: a later retry must not conceal a first local mutation.
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
let error = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.send(),
|
||||
)
|
||||
.await?
|
||||
.expect_err("ordinary deletion succeeded in a managed bucket");
|
||||
let service = error
|
||||
.as_service_error()
|
||||
.ok_or("delete failed without an S3 service rejection")?;
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(500));
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(PUBLIC_INTERNAL_ERROR));
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"first delete through node {node} wrote or rolled back managed data"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_batch_denied(dist: &DistCluster, node: usize, bucket: &str, objects: &[ObjectIdentifier]) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
let response = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(Delete::builder().set_objects(Some(objects.to_vec())).quiet(false).build()?)
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
assert!(response.deleted().is_empty(), "managed batch reported a successful delete");
|
||||
assert_eq!(response.errors().len(), objects.len(), "batch must reject every requested occurrence");
|
||||
for (error, requested) in response.errors().iter().zip(objects) {
|
||||
assert_eq!(error.key(), Some(requested.key()));
|
||||
assert_eq!(error.version_id(), requested.version_id());
|
||||
assert_eq!(error.code(), Some("InternalError"));
|
||||
assert_eq!(error.message(), Some(PUBLIC_INTERNAL_ERROR));
|
||||
}
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"first batch through node {node} wrote or rolled back managed data"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn copy_existing_object(source: &Path, destination: &Path) -> TestResult {
|
||||
assert!(!destination.exists());
|
||||
for entry in walkdir::WalkDir::new(source) {
|
||||
let entry = entry?;
|
||||
let target = destination.join(entry.path().strip_prefix(source)?);
|
||||
if entry.file_type().is_dir() {
|
||||
std::fs::create_dir_all(&target)?;
|
||||
} else {
|
||||
assert!(entry.file_type().is_file());
|
||||
std::fs::copy(entry.path(), target)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ordinary_delete_control(dist: &DistCluster, bucket: &str) -> TestResult {
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
let key = "ordinary-observer-control.bin";
|
||||
let version = Version::put(dist, bucket, key, key).await?;
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let deleted = client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(deleted.delete_marker(), Some(true));
|
||||
let marker = deleted.version_id().ok_or("ordinary delete omitted marker ID")?;
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? != before,
|
||||
"physical observer missed real marker creation"
|
||||
);
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id(marker)
|
||||
.send()
|
||||
.await?;
|
||||
// The logical version inventory is restored, but ctime/identity still
|
||||
// expose the two writes. This checks the observer used by managed negatives.
|
||||
let listed = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
|
||||
assert_eq!(listed.versions().len(), 1);
|
||||
assert_eq!(listed.versions()[0].version_id(), Some(version.id.as_str()));
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? != before,
|
||||
"marker round-trip hid transient physical writes"
|
||||
);
|
||||
assert_exact_version(&client, bucket, key, &version.id, &version.body).await?;
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id(&version.id)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(key)
|
||||
.send()
|
||||
.await?
|
||||
.versions()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let current = Version::put(dist, bucket, "ordinary-batch-current.bin", "current").await?;
|
||||
let exact = Version::put(dist, bucket, "ordinary-batch-exact.bin", "exact").await?;
|
||||
let response = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(target(¤t.key, None)?)
|
||||
.objects(target(&exact.key, Some(&exact.id))?)
|
||||
.objects(target(&exact.key, Some(&exact.id))?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(response.errors().is_empty());
|
||||
assert_eq!(response.deleted().len(), 3);
|
||||
let listed = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(¤t.key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 1);
|
||||
assert_eq!(listed.delete_markers().len(), 1);
|
||||
assert!(
|
||||
client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(&exact.key)
|
||||
.send()
|
||||
.await?
|
||||
.versions()
|
||||
.is_empty()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_delete_entrypoints_preserve_versions_and_physical_state() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (identity, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = identity["bucket"]
|
||||
.as_str()
|
||||
.ok_or("managed receipt omitted bucket")?
|
||||
.to_owned();
|
||||
let prefix = identity["attempt_prefix"]
|
||||
.as_str()
|
||||
.ok_or("managed receipt omitted prefix")?
|
||||
.to_owned();
|
||||
let key = format!("{prefix}native.bin");
|
||||
let mut versions = vec![
|
||||
Version::put(&dist, &bucket, &key, "native-old").await?,
|
||||
Version::put(&dist, &bucket, &key, "native-new").await?,
|
||||
];
|
||||
let ordinary = unique_bucket("delete-control");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary).await?;
|
||||
|
||||
// A new managed bucket cannot legitimately create a marker. Model existing
|
||||
// persisted state from before adoption/upgrade using a real S3-generated
|
||||
// ordinary object+marker, copied disk-for-disk in the same erasure set.
|
||||
// Only this object's files move; bucket authority/configuration is untouched.
|
||||
let marker_key = format!("{prefix}existing-marker.bin");
|
||||
versions.push(Version::put(&dist, &ordinary, &marker_key, "existing-data").await?);
|
||||
let output = dist
|
||||
.client(2)?
|
||||
.delete_object()
|
||||
.bucket(&ordinary)
|
||||
.key(&marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(output.delete_marker(), Some(true));
|
||||
let marker = output.version_id().ok_or("ordinary fixture omitted marker ID")?.to_owned();
|
||||
dist.cluster.stop();
|
||||
for node in &dist.cluster.nodes {
|
||||
let disk = Path::new(&node.data_dirs[0]);
|
||||
copy_existing_object(&disk.join(&ordinary).join(&marker_key), &disk.join(&bucket).join(&marker_key))?;
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
|
||||
ordinary_delete_control(&dist, &ordinary).await?;
|
||||
let batch = vec![
|
||||
target(&key, None)?,
|
||||
target(&key, Some(&versions[0].id))?,
|
||||
target(&key, Some(&versions[1].id))?,
|
||||
target(&marker_key, Some(&marker))?,
|
||||
target(&key, Some(&versions[0].id))?,
|
||||
target(&marker_key, Some(&versions[2].id))?,
|
||||
];
|
||||
for phase in ["open", "sealed", "restarted"] {
|
||||
if phase == "sealed" {
|
||||
confirm_seal(&dist, &admin, &json!({"version": 1, "target": identity}).to_string(), &identity).await?;
|
||||
} else if phase == "restarted" {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_single_denied(&dist, node, &bucket, &key, None).await?;
|
||||
assert_single_denied(&dist, node, &bucket, &key, Some(&versions[0].id)).await?;
|
||||
assert_single_denied(&dist, node, &bucket, &marker_key, Some(&marker)).await?;
|
||||
assert_batch_denied(&dist, node, &bucket, &batch).await?;
|
||||
}
|
||||
assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_delete_test::physical_snapshot;
|
||||
use super::artifact_seal_test::{ArtifactAdmin, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, wait_for_ready};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
signed_s3_request,
|
||||
};
|
||||
use futures::{StreamExt as _, TryStreamExt as _};
|
||||
use http::{Method, StatusCode};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::{collections::BTreeSet, time::Duration};
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
const ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/inventory";
|
||||
const BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
struct Reader {
|
||||
access: String,
|
||||
secret: String,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
async fn create(dist: &DistCluster, target: &Value, action: &str) -> TestResult<Self> {
|
||||
let reader = Self {
|
||||
access: format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]),
|
||||
secret: Uuid::new_v4().simple().to_string(),
|
||||
};
|
||||
let policy_name = format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]);
|
||||
let bucket = target["bucket"].as_str().ok_or("target bucket")?;
|
||||
let prefix = target["attempt_prefix"]
|
||||
.as_str()
|
||||
.ok_or("target prefix")?
|
||||
.trim_end_matches('/');
|
||||
let resource = if action == "s3:ListBucket" {
|
||||
format!("arn:aws:s3:::{bucket}")
|
||||
} else {
|
||||
format!("arn:aws:s3:::{bucket}/{prefix}")
|
||||
};
|
||||
let policy =
|
||||
json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":action,"Resource":resource}]}).to_string();
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, secret, &reader.access, &reader.secret).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &reader.access).await?;
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
async fn request(&self, dist: &DistCluster, target: &Value, invocation: Uuid) -> TestResult<reqwest::Response> {
|
||||
timeout(
|
||||
BUDGET,
|
||||
signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(json!({"version":1,"invocation_id":invocation,"target":target}).to_string()),
|
||||
Some("application/json"),
|
||||
&self.access,
|
||||
&self.secret,
|
||||
),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Frame {
|
||||
version: u32,
|
||||
invocation_id: Uuid,
|
||||
target: Value,
|
||||
sequence: u64,
|
||||
payload: Payload,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
|
||||
enum Payload {
|
||||
Observation {
|
||||
disk_id: Uuid,
|
||||
key: String,
|
||||
version_id: Uuid,
|
||||
delete_marker: bool,
|
||||
},
|
||||
Complete {
|
||||
members: usize,
|
||||
observations: u64,
|
||||
},
|
||||
}
|
||||
|
||||
async fn read_frames(mut response: reqwest::Response) -> TestResult<(Vec<Frame>, bool)> {
|
||||
let mut pending = Vec::new();
|
||||
let mut frames = Vec::new();
|
||||
let mut clean = true;
|
||||
loop {
|
||||
match timeout(BUDGET, response.chunk()).await? {
|
||||
Ok(Some(chunk)) => {
|
||||
pending.extend_from_slice(&chunk);
|
||||
while let Some(end) = pending.iter().position(|byte| *byte == b'\n') {
|
||||
assert!(end < 1024 * 1024, "HTTP inventory line exceeds encoded bound");
|
||||
let line: Vec<_> = pending.drain(..=end).collect();
|
||||
frames.push(serde_json::from_slice(&line[..end])?);
|
||||
}
|
||||
assert!(pending.len() < 1024 * 1024, "unbounded incomplete HTTP line");
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(_) => {
|
||||
clean = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((frames, clean && pending.is_empty()))
|
||||
}
|
||||
|
||||
async fn seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value) -> TestResult {
|
||||
let (status, body) = user
|
||||
.request(
|
||||
dist,
|
||||
0,
|
||||
"/rustfs/admin/v3/plugin-artifacts/seal-writes",
|
||||
json!({"version":1,"target":target}).to_string(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::OK, "actual all-member closure: {body}");
|
||||
assert_eq!(serde_json::from_str::<Value>(&body)?["all_writes_sealed"], true);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn exact_inventory(frames: &[Frame], target: &Value, invocation: Uuid, versions: &BTreeSet<(String, Uuid, bool)>) {
|
||||
let mut observed = BTreeSet::new();
|
||||
let mut disks = BTreeSet::new();
|
||||
let mut count = 0u64;
|
||||
for (sequence, frame) in frames.iter().enumerate() {
|
||||
assert_eq!(frame.version, 1);
|
||||
assert_eq!(frame.invocation_id, invocation);
|
||||
assert_eq!(&frame.target, target);
|
||||
assert_eq!(frame.sequence, u64::try_from(sequence).expect("bounded fixture sequence"));
|
||||
match &frame.payload {
|
||||
Payload::Observation {
|
||||
disk_id,
|
||||
key,
|
||||
version_id,
|
||||
delete_marker,
|
||||
} => {
|
||||
assert!(!disk_id.is_nil() && !version_id.is_nil());
|
||||
assert!(
|
||||
observed.insert((*disk_id, key.clone(), *version_id, *delete_marker)),
|
||||
"duplicate physical observation"
|
||||
);
|
||||
disks.insert(*disk_id);
|
||||
count += 1;
|
||||
}
|
||||
Payload::Complete { members, observations } => {
|
||||
assert_eq!(sequence + 1, frames.len(), "Complete must be the last HTTP frame");
|
||||
assert_eq!(*members, 4);
|
||||
assert_eq!(*observations, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
|
||||
"A03I3A_HTTP_COMPLETE"
|
||||
);
|
||||
if !versions.is_empty() {
|
||||
assert_eq!(disks.len(), 4);
|
||||
}
|
||||
for disk in disks {
|
||||
let actual: BTreeSet<_> = observed
|
||||
.iter()
|
||||
.filter(|(id, ..)| *id == disk)
|
||||
.map(|(_, key, version, marker)| (key.clone(), *version, *marker))
|
||||
.collect();
|
||||
assert_eq!(&actual, versions, "A03I3A_EXACT_VERSIONS");
|
||||
}
|
||||
assert_eq!(observed.len(), versions.len() * 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_exact_versions() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
|
||||
let mut versions = BTreeSet::new();
|
||||
for (name, body) in [
|
||||
("same", b"first".to_vec()),
|
||||
("same", b"second".to_vec()),
|
||||
("zero", Vec::new()),
|
||||
] {
|
||||
let key = format!("{prefix}{name}");
|
||||
let output = timeout(BUDGET, dist.client(1)?.put_object().bucket(bucket).key(&key).body(body.into()).send()).await??;
|
||||
let version = Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?;
|
||||
versions.insert((key, version, false));
|
||||
}
|
||||
seal(&dist, &admin, &target).await?;
|
||||
// A wildcard action is still restricted to this exact attempt resource.
|
||||
// The separate authority case exercises the dedicated action alone.
|
||||
let reader = Reader::create(&dist, &target, "admin:*").await?;
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = reader.request(&dist, &target, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK, "A03I3A_HTTP_ROUTE");
|
||||
assert_eq!(
|
||||
response.headers().get("content-type").and_then(|value| value.to_str().ok()),
|
||||
Some("application/x-ndjson")
|
||||
);
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean, "Complete requires clean HTTP EOF");
|
||||
assert!(frames.len() > 2, "real multi-frame HTTP body");
|
||||
exact_inventory(&frames, &target, invocation, &versions);
|
||||
assert!(
|
||||
before == physical_snapshot(&dist, bucket)?,
|
||||
"inventory mutated physical artifact namespace"
|
||||
);
|
||||
let (empty, empty_admin) = create_artifact_attempt(&dist).await?;
|
||||
seal(&dist, &empty_admin, &empty).await?;
|
||||
let empty_reader = Reader::create(&dist, &empty, "admin:*").await?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = empty_reader.request(&dist, &empty, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean);
|
||||
exact_inventory(&frames, &empty, invocation, &BTreeSet::new());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_scoped_authority() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
seal(&dist, &admin, &target).await?;
|
||||
let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = reader.request(&dist, &target, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK, "dedicated inventory authority");
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean);
|
||||
exact_inventory(&frames, &target, invocation, &BTreeSet::new());
|
||||
let mut other_prefix = target.clone();
|
||||
other_prefix["attempt_prefix"] = json!(format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4()));
|
||||
let mut other_bucket = target.clone();
|
||||
other_bucket["bucket"] = json!("another-artifact-bucket");
|
||||
for denied in [&other_prefix, &other_bucket] {
|
||||
assert_eq!(
|
||||
reader.request(&dist, denied, Uuid::new_v4()).await?.status(),
|
||||
StatusCode::FORBIDDEN,
|
||||
"A03I3A_SCOPED_AUTHORITY"
|
||||
);
|
||||
}
|
||||
let (status, _) = admin
|
||||
.request(
|
||||
&dist,
|
||||
0,
|
||||
ROUTE,
|
||||
json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "A03I3A_SCOPED_AUTHORITY seal-only user");
|
||||
let listed = Reader::create(&dist, &target, "s3:ListBucket").await?;
|
||||
dist.client_with_credentials(0, &listed.access, &listed.secret)?
|
||||
.list_objects_v2()
|
||||
.bucket(target["bucket"].as_str().ok_or("bucket")?)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
listed.request(&dist, &target, Uuid::new_v4()).await?.status(),
|
||||
StatusCode::FORBIDDEN,
|
||||
"A03I3A_SCOPED_AUTHORITY S3 list-only user"
|
||||
);
|
||||
let root_response = signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string()),
|
||||
Some("application/json"),
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(root_response.status(), StatusCode::FORBIDDEN, "original scoped user required");
|
||||
for body in [
|
||||
json!({"version":1,"invocation_id":Uuid::nil(),"target":target}),
|
||||
json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target,"extra":true}),
|
||||
] {
|
||||
let response = signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(body.to_string()),
|
||||
Some("application/json"),
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "strict inventory request");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_incomplete(response: reqwest::Response) -> TestResult<Vec<Frame>> {
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::OK,
|
||||
"valid request must exercise the streaming failure boundary"
|
||||
);
|
||||
let (frames, _) = read_frames(response).await?;
|
||||
assert!(
|
||||
!frames.iter().any(|frame| matches!(frame.payload, Payload::Complete { .. })),
|
||||
"A03I3A_NO_FALSE_COMPLETE"
|
||||
);
|
||||
Ok(frames)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_incomplete_has_no_terminal() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
|
||||
// Real PUTs produce more data than a stalled HTTP/2 flow-control window.
|
||||
futures::stream::iter(0..256)
|
||||
.map(|index| {
|
||||
let client = dist.client(1);
|
||||
let key = format!("{prefix}slow/{index:04}/{}/{}", "a".repeat(190), "b".repeat(190));
|
||||
async move {
|
||||
let output = client?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(vec![1].into())
|
||||
.send()
|
||||
.await?;
|
||||
assert!(!Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?.is_nil());
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
|
||||
}
|
||||
})
|
||||
.buffer_unordered(8)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
seal(&dist, &admin, &target).await?;
|
||||
let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
|
||||
dist.cluster.stop_node(3)?;
|
||||
assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let mut wrong = target.clone();
|
||||
wrong["bucket_incarnation"] = json!(Uuid::new_v4());
|
||||
assert_incomplete(reader.request(&dist, &wrong, Uuid::new_v4()).await?).await?;
|
||||
// The configured endpoint map is ordered. Damage its final receiver, so
|
||||
// earlier real observations are emitted before the physical scan fails.
|
||||
let node = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.max_by_key(|node| format!("{}{}", node.url, node.data_dirs[0]))
|
||||
.ok_or("configured members")?;
|
||||
let metadata = std::path::Path::new(&node.data_dirs[0])
|
||||
.join(bucket)
|
||||
.join(format!("{prefix}slow/0000/{}/{}", "a".repeat(190), "b".repeat(190)))
|
||||
.join("xl.meta");
|
||||
let original = std::fs::read(&metadata)?;
|
||||
std::fs::write(&metadata, b"corrupt inventory metadata")?;
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let outcome = assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await;
|
||||
let after = physical_snapshot(&dist, bucket)?;
|
||||
std::fs::write(&metadata, original)?;
|
||||
let frames = outcome?;
|
||||
assert!(before == after, "failed inventory repaired or mutated a physical object");
|
||||
assert!(
|
||||
frames
|
||||
.iter()
|
||||
.any(|frame| matches!(frame.payload, Payload::Observation { .. })),
|
||||
"late physical failure must follow real HTTP observations"
|
||||
);
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.http2_prior_knowledge()
|
||||
.http2_initial_stream_window_size(1)
|
||||
.http2_initial_connection_window_size(1)
|
||||
.http2_adaptive_window(false)
|
||||
.build()?;
|
||||
let url = format!("{}{ROUTE}", dist.cluster.nodes[0].url);
|
||||
let request = |invocation| {
|
||||
json!({"version":1,"invocation_id":invocation,"target":target})
|
||||
.to_string()
|
||||
.into_bytes()
|
||||
};
|
||||
let response = crate::common::signed_request_with_client(
|
||||
&client,
|
||||
Method::POST,
|
||||
&url,
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
Some(request(Uuid::new_v4())),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
// Do not poll the body: its one-byte HTTP/2 receive window stalls transport.
|
||||
// This waits beyond the fixed30s service budget, not for readiness.
|
||||
tokio::time::sleep(Duration::from_secs(35)).await;
|
||||
assert_incomplete(response).await?;
|
||||
let response = crate::common::signed_request_with_client(
|
||||
&client,
|
||||
Method::POST,
|
||||
&url,
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
Some(request(Uuid::new_v4())),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
drop(response);
|
||||
let response = timeout(Duration::from_secs(10), reader.request(&dist, &target, Uuid::new_v4())).await??;
|
||||
let (frames, clean) = timeout(Duration::from_secs(10), read_frames(response)).await??;
|
||||
assert!(
|
||||
clean && matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
|
||||
"fresh scan must succeed promptly after an actual disconnected HTTP body"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_repair_tail_test::PhysicalBarrier;
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
|
||||
use crate::chaos::census_object_version_on_disk;
|
||||
use crate::common::init_logging;
|
||||
use crate::storage_api::artifact_rpc::{
|
||||
DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_protos::proto_gen::node_service::{RenameDataRequest, node_service_client::NodeServiceClient};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PUT_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
|
||||
|
||||
fn signed_put_request(body: RenameDataRequest, audience: &str) -> TestResult<Request<RenameDataRequest>> {
|
||||
let canonical = rustfs_protos::canonical_artifact_rename_data_request_body(&body)?;
|
||||
let mut request = Request::new(body);
|
||||
request.set_timeout(Duration::from_secs(15));
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.ok_or("missing canonical digest")?
|
||||
.to_str()?
|
||||
.to_owned();
|
||||
request.metadata_mut().as_mut().extend(gen_tonic_signature_headers(
|
||||
audience,
|
||||
TONIC_RPC_PREFIX.trim_start_matches('/'),
|
||||
"ArtifactRenameData",
|
||||
Some(&digest),
|
||||
)?);
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
fn publication_snapshot(paths: &[&Path]) -> TestResult<BTreeMap<PathBuf, Option<Vec<u8>>>> {
|
||||
let mut snapshot = BTreeMap::new();
|
||||
for root in paths {
|
||||
for entry in walkdir::WalkDir::new(root) {
|
||||
let entry = entry?;
|
||||
let contents = if entry.file_type().is_dir() {
|
||||
None
|
||||
} else {
|
||||
Some(std::fs::read(entry.path())?)
|
||||
};
|
||||
snapshot.insert(entry.path().to_path_buf(), contents);
|
||||
}
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn assert_raw_put_closed(body: &RenameDataRequest, url: &str, audience: &str, paths: &[&Path]) -> TestResult {
|
||||
let before = publication_snapshot(paths)?;
|
||||
// Re-sign the exact original request; replay protection is not the rejection.
|
||||
let request = signed_put_request(body.clone(), audience)?;
|
||||
let response = timeout(Duration::from_secs(20), async {
|
||||
let mut client = NodeServiceClient::connect(url.to_owned()).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?.into_inner())
|
||||
})
|
||||
.await??;
|
||||
assert!(!response.success, "sealed attempt admitted another ordinary PUT");
|
||||
let error = response.error.ok_or("rejected raw PUT omitted error")?;
|
||||
assert_eq!(error.code, DiskError::Io(std::io::ErrorKind::PermissionDenied.into()).to_u32());
|
||||
assert_eq!(error.error_info, "io error artifact attempt mutation domain is closed");
|
||||
assert_eq!(publication_snapshot(paths)?, before, "rejected raw PUT mutated staged or published files");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_put_seal_waits_for_disconnected_physical_tail() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
|
||||
let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let key = format!("{prefix}detached-put.bin");
|
||||
let payload = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(payload.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?;
|
||||
let request_body = json!({"version": 1, "target": target}).to_string();
|
||||
let baselines = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, &key, Some(version)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
assert!(
|
||||
baselines
|
||||
.iter()
|
||||
.all(|baseline| baseline.is_complete() && !baseline.expected_part_numbers.is_empty())
|
||||
);
|
||||
|
||||
let node = &dist.cluster.nodes[3];
|
||||
let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
|
||||
let object = disk.join(bucket).join(&key);
|
||||
let metadata_bytes = std::fs::read(object.join("xl.meta"))?;
|
||||
let fi = FileMeta::load(&metadata_bytes)?.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
let admitted_file_info = serde_json::to_value(&fi)?;
|
||||
assert!(!fi.is_healing(), "ordinary publication must not carry the repair marker");
|
||||
let data_dir = fi
|
||||
.data_dir
|
||||
.ok_or("physical-tail fixture requires external shards")?
|
||||
.to_string();
|
||||
let source = Uuid::new_v4().to_string();
|
||||
let staging = disk.join(".rustfs.sys/tmp").join(&source);
|
||||
std::fs::create_dir_all(&staging)?;
|
||||
let destination = object.join(&data_dir);
|
||||
// Replay the receiver publication of a real ordinary PUT while its attempt
|
||||
// remains open. Preserve its referenced version metadata. The admitted
|
||||
// owned operation may still finish metadata publication after RPC detach.
|
||||
std::fs::rename(&destination, staging.join(&data_dir))?;
|
||||
let barrier = PhysicalBarrier::arm(&disk, &destination)?;
|
||||
let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
|
||||
let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
|
||||
let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
|
||||
let put = RenameDataRequest {
|
||||
disk: disk_id.to_string(),
|
||||
src_volume: ".rustfs.sys/tmp".to_owned(),
|
||||
src_path: source,
|
||||
dst_volume: bucket.to_owned(),
|
||||
dst_path: key.clone(),
|
||||
file_info: serde_json::to_string(&fi)?,
|
||||
file_info_bin: Default::default(),
|
||||
scanner_publication_lease_token: Default::default(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
};
|
||||
let request = signed_put_request(put.clone(), &node.address)?;
|
||||
// Direct RPC takes no coordinator attempt lock and owns its own connection.
|
||||
let url = node.url.clone();
|
||||
let mut rpc = tokio::spawn(async move {
|
||||
timeout(Duration::from_secs(30), async move {
|
||||
let mut client = NodeServiceClient::connect(url).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?)
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::select! {
|
||||
result = &mut rpc => {
|
||||
let response = result???.into_inner();
|
||||
panic!(
|
||||
"ordinary PUT receiver completed before the physical barrier: success={}, error={:?}",
|
||||
response.success, response.error
|
||||
);
|
||||
}
|
||||
arrived = barrier.observed("arrived") => arrived?,
|
||||
}
|
||||
assert!(!destination.exists(), "physical PUT escaped its barrier");
|
||||
rpc.abort();
|
||||
assert!(rpc.await.expect_err("disconnect the ordinary PUT waiter").is_cancelled());
|
||||
// The ordinary PUT wrapper retains an owned task. Observe its RPC-facing
|
||||
// waiter dropping, rather than cancellation of the continuing disk task.
|
||||
barrier.observed("rpc-waiter-dropped").await?;
|
||||
|
||||
let seal = user.request(&dist, 0, PUT_SEAL_ROUTE, request_body.clone());
|
||||
tokio::pin!(seal);
|
||||
let closed = wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async {
|
||||
Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-put-seals"))
|
||||
.min_depth(2)
|
||||
.max_depth(2)
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| entry.file_name() == "closed"))
|
||||
},
|
||||
"PUT receiver persisted closing before physical drain",
|
||||
);
|
||||
tokio::pin!(closed);
|
||||
tokio::select! {
|
||||
result = &mut seal => {
|
||||
let (status, response) = result?;
|
||||
panic!("PUT seal returned before its blocked receiver closed admission: {status} {response}");
|
||||
}
|
||||
result = &mut closed => result?,
|
||||
}
|
||||
let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"PUT seal confirmed an unfinished physical tail: {response}"
|
||||
);
|
||||
assert!(response.contains("artifact operation unconfirmed"));
|
||||
assert!(!destination.exists());
|
||||
assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
|
||||
assert!(!destination.exists(), "a second admission published while the worker was paused");
|
||||
|
||||
barrier.release()?;
|
||||
wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async { Ok(destination.join("part.1").is_file()) },
|
||||
"detached ordinary PUT worker finished",
|
||||
)
|
||||
.await?;
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
// Seal confirmation is the boundary: already admitted work may re-encode
|
||||
// metadata before this point, but the complete FileInfo must be unchanged.
|
||||
let published_objects: Vec<_> = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| PathBuf::from(&node.data_dirs[0]).join(bucket).join(&key))
|
||||
.collect();
|
||||
let published_paths: Vec<_> = published_objects.iter().map(PathBuf::as_path).collect();
|
||||
let sealed_snapshot = publication_snapshot(&published_paths)?;
|
||||
let sealed_file_info =
|
||||
FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
assert_eq!(
|
||||
serde_json::to_value(sealed_file_info)?,
|
||||
admitted_file_info,
|
||||
"admitted PUT changed FileInfo semantics"
|
||||
);
|
||||
for (index, baseline) in baselines.iter().enumerate() {
|
||||
assert!(
|
||||
census_object_version_on_disk(Path::new(&dist.cluster.nodes[index].data_dirs[0]), bucket, &key, Some(version))?
|
||||
.matches_manifest(baseline)
|
||||
);
|
||||
assert_exact_version(&dist.client(index)?, bucket, &key, version, &payload).await?;
|
||||
}
|
||||
|
||||
// Stage the exact request again. A source metadata sentinel makes a missing
|
||||
// admission check observable even if the existing destination later causes
|
||||
// ENOTEMPTY: ordinary rename rewrites source metadata before that syscall.
|
||||
let staged_data = staging.join(&data_dir);
|
||||
std::fs::create_dir_all(&staged_data)?;
|
||||
for entry in std::fs::read_dir(&destination)? {
|
||||
let entry = entry?;
|
||||
assert!(entry.file_type()?.is_file(), "unexpected nested shard fixture");
|
||||
std::fs::copy(entry.path(), staged_data.join(entry.file_name()))?;
|
||||
}
|
||||
std::fs::write(staging.join("xl.meta"), b"closed-admission-must-preserve-this-staged-metadata")?;
|
||||
assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(&key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 1, "raw PUT retries created another version");
|
||||
assert_eq!(listed.versions()[0].version_id(), Some(version));
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
assert!(
|
||||
publication_snapshot(&published_paths)? == sealed_snapshot,
|
||||
"published files changed after the first successful physical-drain confirmation"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
|
||||
use crate::chaos::census_object_version_on_disk;
|
||||
use crate::common::init_logging;
|
||||
use crate::storage_api::artifact_rpc::{TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
|
||||
use http::StatusCode;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
ArtifactRepairRenameDataRequest, RenameDataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use uuid::Uuid;
|
||||
|
||||
const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
|
||||
|
||||
pub(super) struct PhysicalBarrier(PathBuf);
|
||||
|
||||
impl PhysicalBarrier {
|
||||
pub(super) fn arm(disk: &Path, destination: &Path) -> TestResult<Self> {
|
||||
let path = disk.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
std::fs::write(&path, destination.to_str().ok_or("non-UTF8 fixture destination")?)?;
|
||||
Ok(Self(path))
|
||||
}
|
||||
|
||||
pub(super) async fn observed(&self, stage: &str) -> TestResult {
|
||||
wait_until(Duration::from_secs(15), || async { Ok(self.0.with_extension(stage).is_file()) }, stage).await
|
||||
}
|
||||
|
||||
pub(super) fn release(&self) -> TestResult {
|
||||
std::fs::write(self.0.with_extension("release"), b"release physical worker")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PhysicalBarrier {
|
||||
fn drop(&mut self) {
|
||||
// Release even on assertion failure; the worker also has its own deadline.
|
||||
let _ = self.release();
|
||||
let _ = std::fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_waits_for_disconnected_physical_tail() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
// Match the other raw internode signature tests sharing the OnceLock.
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
|
||||
let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let key = format!("{prefix}detached-rename.bin");
|
||||
let body = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?;
|
||||
let request_body = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
|
||||
let node = &dist.cluster.nodes[3];
|
||||
let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
|
||||
let object = disk.join(bucket).join(&key);
|
||||
let metadata = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
|
||||
let mut fi = metadata.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
fi.set_healing();
|
||||
let data_dir = fi
|
||||
.data_dir
|
||||
.ok_or("physical-tail fixture requires external shard files")?
|
||||
.to_string();
|
||||
let baseline = census_object_version_on_disk(&disk, bucket, &key, Some(version))?;
|
||||
assert!(baseline.is_complete() && !baseline.expected_part_numbers.is_empty());
|
||||
let source = Uuid::new_v4().to_string();
|
||||
let staging = disk.join(".rustfs.sys/tmp").join(&source);
|
||||
std::fs::create_dir_all(&staging)?;
|
||||
let destination = object.join(&data_dir);
|
||||
std::fs::rename(&destination, staging.join(&data_dir))?;
|
||||
let barrier = PhysicalBarrier::arm(&disk, &destination)?;
|
||||
let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
|
||||
let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
|
||||
let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
|
||||
let repair = ArtifactRepairRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: disk_id.to_string(),
|
||||
src_volume: ".rustfs.sys/tmp".to_owned(),
|
||||
src_path: source,
|
||||
dst_volume: bucket.to_owned(),
|
||||
dst_path: key.clone(),
|
||||
file_info: serde_json::to_string(&fi)?,
|
||||
file_info_bin: Default::default(),
|
||||
scanner_publication_lease_token: Default::default(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&repair)?;
|
||||
let mut request = Request::new(repair);
|
||||
request.set_timeout(Duration::from_secs(15));
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.ok_or("missing canonical digest")?
|
||||
.to_str()?
|
||||
.to_owned();
|
||||
let headers = gen_tonic_signature_headers(
|
||||
&node.address,
|
||||
TONIC_RPC_PREFIX.trim_start_matches('/'),
|
||||
"ArtifactRepairRenameData",
|
||||
Some(&digest),
|
||||
)?;
|
||||
request.metadata_mut().as_mut().extend(headers);
|
||||
// An independent channel and direct receiver RPC take no coordinator attempt
|
||||
// lock. Dropping this task closes the client stream while rename is paused.
|
||||
let url = node.url.clone();
|
||||
let mut rpc = tokio::spawn(async move {
|
||||
timeout(Duration::from_secs(30), async move {
|
||||
let mut client = NodeServiceClient::connect(url).await?;
|
||||
client
|
||||
.artifact_repair_rename_data(request)
|
||||
.await
|
||||
.map_err(|error| -> Box<dyn std::error::Error + Send + Sync> { Box::new(error) })
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::select! {
|
||||
result = &mut rpc => {
|
||||
let response = result???.into_inner();
|
||||
panic!(
|
||||
"repair receiver completed before the physical barrier: success={}, error={:?}",
|
||||
response.success, response.error
|
||||
);
|
||||
}
|
||||
arrived = barrier.observed("arrived") => arrived?,
|
||||
}
|
||||
assert!(!destination.exists(), "physical publication ran before its barrier");
|
||||
rpc.abort();
|
||||
assert!(rpc.await.expect_err("cancel the disconnected RPC waiter").is_cancelled());
|
||||
// This acknowledgement is emitted by the receiver's dropped async waiter,
|
||||
// not by the client and not by a delayed response.
|
||||
barrier.observed("waiter-dropped").await?;
|
||||
|
||||
let seal = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone());
|
||||
tokio::pin!(seal);
|
||||
let closed = wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async {
|
||||
Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-repair-seals"))
|
||||
.min_depth(2)
|
||||
.max_depth(2)
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| entry.file_name() == "closed"))
|
||||
},
|
||||
"receiver persisted repair closure while its physical worker remains paused",
|
||||
);
|
||||
tokio::pin!(closed);
|
||||
tokio::select! {
|
||||
result = &mut seal => {
|
||||
let (status, response) = result?;
|
||||
panic!("repair seal returned before the blocked receiver closed admission: {status} {response}");
|
||||
}
|
||||
result = &mut closed => result?,
|
||||
}
|
||||
// Cover a complete receiver drain timeout, so early physical-owner release
|
||||
// causes a successful response here and fails this runtime assertion.
|
||||
let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"seal confirmed an unfinished physical tail: {response}"
|
||||
);
|
||||
assert!(response.contains("artifact operation unconfirmed"));
|
||||
assert!(!destination.exists(), "worker escaped the physical barrier");
|
||||
barrier.release()?;
|
||||
wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async { Ok(destination.join("part.1").is_file()) },
|
||||
"detached physical rename completes after release",
|
||||
)
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(45),
|
||||
|| async {
|
||||
let (status, response) = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone()).await?;
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed") {
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "repair seal did not confirm after physical drain: {response}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&response)?,
|
||||
json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"exact retry confirms all four receivers after physical drain",
|
||||
)
|
||||
.await?;
|
||||
assert!(census_object_version_on_disk(&disk, bucket, &key, Some(version))?.matches_manifest(&baseline));
|
||||
assert_exact_version(&dist.client(0)?, bucket, &key, version, &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, payload_for, unique_bucket, wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post, start_root_heal_when_control_ready};
|
||||
use crate::common::init_logging;
|
||||
use http::StatusCode;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
|
||||
const HEAL_BODY: &str =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
const HEAL_BUDGET: Duration = Duration::from_secs(120);
|
||||
|
||||
pub(super) struct RepairObject {
|
||||
pub(super) key: String,
|
||||
pub(super) version: String,
|
||||
pub(super) body: Vec<u8>,
|
||||
manifests: Vec<VersionShardCensus>,
|
||||
}
|
||||
|
||||
fn roots(dist: &DistCluster) -> Vec<PathBuf> {
|
||||
dist.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| PathBuf::from(&node.data_dirs[0]))
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl RepairObject {
|
||||
pub(super) async fn put(dist: &DistCluster, bucket: &str, key: String) -> TestResult<Self> {
|
||||
let body = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?.to_owned();
|
||||
let manifests = roots(dist)
|
||||
.iter()
|
||||
.map(|disk| census_object_version_on_disk(disk, bucket, &key, Some(&version)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
for manifest in &manifests {
|
||||
assert!(manifest.is_complete(), "repair needs a complete baseline: {manifest:?}");
|
||||
assert!(!manifest.expected_part_numbers.is_empty(), "repair must use actual shard files");
|
||||
}
|
||||
Ok(Self {
|
||||
key,
|
||||
version,
|
||||
body,
|
||||
manifests,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn part(&self, disk: &Path, bucket: &str, node: usize) -> TestResult<PathBuf> {
|
||||
let data_dir = self.manifests[node]
|
||||
.data_dir
|
||||
.as_ref()
|
||||
.ok_or("external shard omitted data directory")?;
|
||||
Ok(disk.join(bucket).join(&self.key).join(data_dir).join("part.1"))
|
||||
}
|
||||
|
||||
pub(super) async fn assert_repaired(&self, dist: &DistCluster, bucket: &str) -> TestResult {
|
||||
wait_until(
|
||||
HEAL_BUDGET,
|
||||
|| async {
|
||||
for (disk, baseline) in roots(dist).iter().zip(&self.manifests) {
|
||||
if !census_object_version_on_disk(disk, bucket, &self.key, Some(&self.version))?.matches_manifest(baseline) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"every configured receiver restored the exact version and original shard fingerprints",
|
||||
)
|
||||
.await?;
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_exact_version(&dist.client(node)?, bucket, &self.key, &self.version, &self.body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn start_heal(dist: &DistCluster, bucket: &str, prefix: &str, remove: bool) -> TestResult<String> {
|
||||
let url = format!("{}/rustfs/admin/v3/heal/{bucket}/{prefix}?forceStart=true", dist.cluster.nodes[0].url);
|
||||
let body = if remove {
|
||||
HEAL_BODY.replace("\"remove\":false", "\"remove\":true")
|
||||
} else {
|
||||
HEAL_BODY.to_owned()
|
||||
};
|
||||
let response = timeout(
|
||||
HEAL_BUDGET,
|
||||
start_root_heal_when_control_ready(&url, &body, &dist.cluster.access_key, &dist.cluster.secret_key),
|
||||
)
|
||||
.await??;
|
||||
let response: Value = serde_json::from_str(&response)?;
|
||||
Ok(response["clientToken"]
|
||||
.as_str()
|
||||
.ok_or("heal omitted its task token")?
|
||||
.to_owned())
|
||||
}
|
||||
|
||||
pub(super) async fn finish_heal(dist: &DistCluster, token: &str, rejected_key: Option<&str>) -> TestResult {
|
||||
let url = format!("{}/rustfs/admin/v3/heal/?clientToken={token}", dist.cluster.nodes[0].url);
|
||||
wait_until(
|
||||
HEAL_BUDGET,
|
||||
|| async {
|
||||
let response = signed_admin_post(&url, None, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
let status: Value = serde_json::from_str(&response)?;
|
||||
if status["summary"] == "running" {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Some(key) = rejected_key {
|
||||
assert_eq!(status["summary"], "stopped", "closed repair must report failure: {status}");
|
||||
assert!(
|
||||
status["outcome"]["counters"]["failed"]
|
||||
.as_u64()
|
||||
.is_some_and(|count| count > 0)
|
||||
);
|
||||
assert!(
|
||||
status["outcome"]["objects"].as_array().is_some_and(|objects| objects
|
||||
.iter()
|
||||
.any(|object| { object["identity"]["object"] == key && object["disposition"]["state"] == "failed" })),
|
||||
"heal must actually attempt the rejected object: {status}"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(status["summary"], "finished", "heal did not complete: {status}");
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"exact admin heal task reached its expected terminal outcome",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn repair_seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
|
||||
wait_until(
|
||||
Duration::from_secs(45),
|
||||
|| async {
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
let (status, body) = user.request(dist, 0, REPAIR_SEAL_ROUTE, request).await?;
|
||||
if !confirmed {
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver repair seal: {body}");
|
||||
assert!(body.contains("<Code>ServiceUnavailable</Code>"));
|
||||
assert!(body.contains("artifact operation unconfirmed"));
|
||||
return Ok(true);
|
||||
}
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE
|
||||
&& body.contains("<Code>ServiceUnavailable</Code>")
|
||||
&& body.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "repair seal failed: {body}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&body)?,
|
||||
json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"repair seal confirms every configured receiver",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_shard_and_minority_metadata_repair_after_put_seal() -> TestResult {
|
||||
init_logging();
|
||||
// Sub-quorum metadata recovery supports genuine legacy shards only.
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
|
||||
let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
|
||||
let later = dist
|
||||
.client(2)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&shard.key)
|
||||
.body(b"newer-version".to_vec().into())
|
||||
.send()
|
||||
.await?;
|
||||
let later_version = later.version_id().ok_or("second PUT omitted version ID")?.to_owned();
|
||||
assert_ne!(shard.version, later_version);
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
|
||||
dist.cluster.stop();
|
||||
let disks = roots(&dist);
|
||||
std::fs::remove_file(shard.part(&disks[3], bucket, 3)?)?;
|
||||
for disk in &disks[1..] {
|
||||
std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
|
||||
}
|
||||
assert!(!census_object_version_on_disk(&disks[3], bucket, &shard.key, Some(&shard.version))?.is_complete());
|
||||
assert!(disks[0].join(bucket).join(&metadata.key).join("xl.meta").is_file());
|
||||
assert!(
|
||||
disks[1..]
|
||||
.iter()
|
||||
.all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
|
||||
);
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let token = start_heal(&dist, bucket, prefix, false).await?;
|
||||
shard.assert_repaired(&dist, bucket).await?;
|
||||
metadata.assert_repaired(&dist, bucket).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
assert_exact_version(&dist.client(3)?, bucket, &shard.key, &later_version, b"newer-version").await?;
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 3, "repair must not publish replacement versions");
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_denies_shard_and_metadata_repair_after_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
|
||||
let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
repair_seal(&dist, &user, &target, true).await?;
|
||||
let ordinary_bucket = unique_bucket("repair-control");
|
||||
dist.create_bucket(&ordinary_bucket).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary_bucket).await?;
|
||||
let ordinary = RepairObject::put(&dist, &ordinary_bucket, "ordinary.bin".to_owned()).await?;
|
||||
let disks = roots(&dist);
|
||||
let part = shard.part(&disks[3], bucket, 3)?;
|
||||
std::fs::remove_file(&part)?;
|
||||
for disk in &disks[1..] {
|
||||
std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
|
||||
}
|
||||
for restarted in [false, true] {
|
||||
if restarted {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
// The same receivers must still execute an ordinary bucket repair.
|
||||
std::fs::remove_file(ordinary.part(&disks[3], &ordinary_bucket, 3)?)?;
|
||||
let control = start_heal(&dist, &ordinary_bucket, &ordinary.key, false).await?;
|
||||
ordinary.assert_repaired(&dist, &ordinary_bucket).await?;
|
||||
finish_heal(&dist, &control, None).await?;
|
||||
for object in [&shard, &metadata] {
|
||||
let token = start_heal(&dist, bucket, &object.key, false).await?;
|
||||
finish_heal(&dist, &token, Some(&object.key)).await?;
|
||||
}
|
||||
assert!(!part.exists(), "sealed repair recreated the missing shard");
|
||||
assert!(
|
||||
disks[1..]
|
||||
.iter()
|
||||
.all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
|
||||
);
|
||||
assert_exact_version(&dist.client(2)?, bucket, &shard.key, &shard.version, &shard.body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_requires_every_receiver_on_exact_retry() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
// Repeat after global confirmation: its persisted receipt cannot replace a
|
||||
// fresh acknowledgement from an offline configured receiver.
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
repair_seal(&dist, &user, &target, false).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
repair_seal(&dist, &user, &target, true).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_dangling_repair_preserves_surviving_metadata() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let object = RepairObject::put(&dist, bucket, format!("{prefix}dangling.bin")).await?;
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
dist.cluster.stop();
|
||||
let disks = roots(&dist);
|
||||
let survivor = disks[0].join(bucket).join(&object.key).join("xl.meta");
|
||||
let before = std::fs::read(&survivor)?;
|
||||
for (node, disk) in disks.iter().enumerate() {
|
||||
std::fs::remove_file(object.part(disk, bucket, node)?)?;
|
||||
if node != 0 {
|
||||
std::fs::remove_file(disk.join(bucket).join(&object.key).join("xl.meta"))?;
|
||||
}
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let token = start_heal(&dist, bucket, &object.key, true).await?;
|
||||
finish_heal(&dist, &token, Some(&object.key)).await?;
|
||||
assert_eq!(
|
||||
std::fs::read(survivor)?,
|
||||
before,
|
||||
"distributed repair must not authorize managed dangling deletion"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, put_object, s3_probe_client, unique_bucket, wait_for_ready,
|
||||
wait_until,
|
||||
};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, admin_request, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use http::{Method, StatusCode};
|
||||
use serde_json::{Value, json};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
const BUCKET_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets";
|
||||
const INSPECT_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets/inspect";
|
||||
const SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
|
||||
const ADMIN_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
pub(super) struct ArtifactAdmin {
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
}
|
||||
|
||||
impl ArtifactAdmin {
|
||||
pub(super) async fn request(
|
||||
&self,
|
||||
dist: &DistCluster,
|
||||
node: usize,
|
||||
route: &str,
|
||||
body: String,
|
||||
) -> TestResult<(StatusCode, String)> {
|
||||
timeout(
|
||||
ADMIN_BUDGET,
|
||||
admin_request(
|
||||
&dist.cluster.nodes[node].url,
|
||||
Method::POST,
|
||||
route,
|
||||
Some(body),
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_artifact_admin(dist: &DistCluster, bucket: &str, prefix: &str) -> TestResult<ArtifactAdmin> {
|
||||
let user = ArtifactAdmin {
|
||||
access_key: format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]),
|
||||
secret_key: Uuid::new_v4().simple().to_string(),
|
||||
};
|
||||
let policy_name = format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]);
|
||||
let policy = json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["admin:CreatePluginArtifactBucket", "admin:GetPluginArtifactBucket"],
|
||||
"Resource": format!("arn:aws:s3:::{bucket}"),
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["admin:SealPluginArtifactAttempt", "admin:SealPluginArtifactRepairs", "admin:SealPluginArtifactWrites"],
|
||||
"Resource": format!("arn:aws:s3:::{bucket}/{}", prefix.trim_end_matches('/')),
|
||||
},
|
||||
],
|
||||
})
|
||||
.to_string();
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, secret, &user.access_key, &user.secret_key).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &user.access_key).await?;
|
||||
Ok(user)
|
||||
}
|
||||
|
||||
pub(super) async fn create_artifact_attempt(dist: &DistCluster) -> TestResult<(Value, ArtifactAdmin)> {
|
||||
let bucket = unique_bucket("artifact");
|
||||
let prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
let user = create_artifact_admin(dist, &bucket, &prefix).await?;
|
||||
let body = json!({"version": 1, "bucket": bucket, "base_prefix": "outputs/"}).to_string();
|
||||
let (status, response) = user.request(dist, 0, BUCKET_ROUTE, body).await?;
|
||||
assert_eq!(status, StatusCode::OK, "artifact bucket creation failed: {response}");
|
||||
let receipt: Value = serde_json::from_str(&response)?;
|
||||
assert_eq!(receipt["version"], 1);
|
||||
assert_eq!(receipt["bucket"], bucket);
|
||||
assert_eq!(receipt["base_prefix"], "outputs/");
|
||||
assert_eq!(receipt["versioning_enabled"], true);
|
||||
let incarnation = receipt["bucket_incarnation"]
|
||||
.as_str()
|
||||
.ok_or("artifact receipt omitted incarnation")?;
|
||||
assert!(!Uuid::parse_str(incarnation)?.is_nil());
|
||||
let target = json!({
|
||||
"bucket": bucket,
|
||||
"bucket_incarnation": incarnation,
|
||||
"attempt_prefix": prefix,
|
||||
});
|
||||
assert_bucket_identity(dist, &user, 1, &target).await?;
|
||||
Ok((target, user))
|
||||
}
|
||||
|
||||
async fn assert_bucket_identity(dist: &DistCluster, user: &ArtifactAdmin, node: usize, target: &Value) -> TestResult {
|
||||
let body = json!({"version": 1, "bucket": target["bucket"], "base_prefix": "outputs/"}).to_string();
|
||||
wait_until(
|
||||
ADMIN_BUDGET,
|
||||
|| async {
|
||||
let (status, response) = user.request(dist, node, INSPECT_ROUTE, body.clone()).await?;
|
||||
// IAM peers may not have received the new user and policy yet.
|
||||
// Storage failures must fail immediately, including the old gate's 503.
|
||||
if matches!(status, StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED)
|
||||
&& (response.contains("<Code>AccessDenied</Code>") || response.contains("<Code>InvalidAccessKeyId</Code>"))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "cross-node artifact inspect failed: {response}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&response)?,
|
||||
json!({
|
||||
"version": 1,
|
||||
"bucket": target["bucket"],
|
||||
"base_prefix": "outputs/",
|
||||
"bucket_incarnation": target["bucket_incarnation"],
|
||||
"versioning_enabled": true,
|
||||
}),
|
||||
"inspect must return the created managed bucket identity"
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"artifact IAM user and policy propagation",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn confirm_seal(dist: &DistCluster, user: &ArtifactAdmin, request: &str, target: &Value) -> TestResult {
|
||||
wait_until(
|
||||
ADMIN_BUDGET,
|
||||
|| async {
|
||||
let (status, response) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
|
||||
// HTTP readiness does not confirm that remote disk handles have
|
||||
// recovered. Retry only the API's explicit same-identity response.
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE
|
||||
&& response.contains("<Code>ServiceUnavailable</Code>")
|
||||
&& response.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "artifact seal failed: {response}");
|
||||
let receipt: Value = serde_json::from_str(&response)?;
|
||||
assert_eq!(receipt, json!({"version": 1, "target": target, "ordinary_put_sealed": true}));
|
||||
Ok(true)
|
||||
},
|
||||
"exact artifact seal confirmation after receiver recovery",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn assert_seal_unconfirmed(dist: &DistCluster, user: &ArtifactAdmin, request: &str) -> TestResult {
|
||||
let (status, body) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver seal response: {body}");
|
||||
assert!(
|
||||
body.contains("<Code>ServiceUnavailable</Code>"),
|
||||
"seal failure lost its service code: {body}"
|
||||
);
|
||||
assert!(
|
||||
body.contains("artifact operation unconfirmed"),
|
||||
"seal failure lost its retry contract: {body}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn assert_exact_version(client: &Client, bucket: &str, key: &str, version: &str, body: &[u8]) -> TestResult {
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.get_object().bucket(bucket).key(key).version_id(version).send(),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(output.version_id(), Some(version));
|
||||
let actual = timeout(Duration::from_secs(30), output.body.collect()).await??.into_bytes();
|
||||
assert_eq!(actual.as_ref(), body);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_put_denied(client: &Client, ordinary_bucket: &str, bucket: &str, key: &str) -> TestResult {
|
||||
// The same receiver must still accept a normal write before its sealed probe.
|
||||
put_object(client, ordinary_bucket, key, b"ordinary-control".to_vec()).await?;
|
||||
assert_object_bytes(client, ordinary_bucket, key, b"ordinary-control").await?;
|
||||
let client = s3_probe_client(client);
|
||||
let result = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(b"must-not-publish".to_vec().into())
|
||||
.send(),
|
||||
)
|
||||
.await?;
|
||||
let error = result.expect_err("sealed artifact attempt accepted a new PUT");
|
||||
let service = error
|
||||
.as_service_error()
|
||||
.ok_or("sealed PUT failed without a service rejection")?;
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(500),
|
||||
"sealed PUT must return its current HTTP status"
|
||||
);
|
||||
assert_eq!(service.code(), Some("InternalError"), "sealed PUT must return its current service code");
|
||||
// StorageError::Io is redacted at the API boundary; the sealed reason is
|
||||
// not exposed. Success controls and exact inventory checks prove the effect.
|
||||
assert_eq!(
|
||||
service.message(),
|
||||
Some("We encountered an internal error, please try again."),
|
||||
"sealed PUT must retain the public redacted message"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_put_seal_survives_full_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
|
||||
let key = format!("{prefix}result.bin");
|
||||
let mut versions = Vec::new();
|
||||
for (node, body) in [(1, b"first-version".as_slice()), (2, b"second-version".as_slice())] {
|
||||
let client = dist.client(node)?;
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.put_object().bucket(bucket).key(&key).body(body.to_vec().into()).send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
|
||||
assert!(!Uuid::parse_str(&version)?.is_nil());
|
||||
assert_exact_version(&dist.client(3)?, bucket, &key, &version, body).await?;
|
||||
versions.push((version, body.to_vec()));
|
||||
}
|
||||
assert_ne!(versions[0].0, versions[1].0);
|
||||
let ordinary = unique_bucket("ordinary");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
put_object(&dist.client(2)?, &ordinary, "normal.bin", b"before-seal".to_vec()).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &ordinary, "normal.bin", b"before-seal").await?;
|
||||
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
let other_prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
for phase in ["sealed", "restarted"] {
|
||||
if phase == "restarted" {
|
||||
dist.cluster.stop();
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
}
|
||||
assert_bucket_identity(&dist, &user, 2, &target).await?;
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
// A second attempt in the managed bucket must remain writable;
|
||||
// a broken bucket-wide write path cannot stand in for sealing.
|
||||
let client = dist.client(node)?;
|
||||
let other_key = format!("{other_prefix}{phase}-{node}.bin");
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&other_key)
|
||||
.body(b"other-attempt".to_vec().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("other attempt PUT omitted version ID")?;
|
||||
assert!(!Uuid::parse_str(version)?.is_nil());
|
||||
assert_exact_version(
|
||||
&dist.client((node + 1) % dist.cluster.nodes.len())?,
|
||||
bucket,
|
||||
&other_key,
|
||||
version,
|
||||
b"other-attempt",
|
||||
)
|
||||
.await?;
|
||||
assert_put_denied(&dist.client(node)?, &ordinary, bucket, &format!("{prefix}{phase}-{node}.bin")).await?;
|
||||
for (version, body) in &versions {
|
||||
assert_exact_version(&dist.client(node)?, bucket, &key, version, body).await?;
|
||||
}
|
||||
}
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 2, "denied PUT left an artifact version: {listed:?}");
|
||||
assert!(listed.delete_markers().is_empty(), "artifact seal created a delete marker");
|
||||
put_object(&dist.client(1)?, &ordinary, "normal.bin", phase.as_bytes().to_vec()).await?;
|
||||
assert_object_bytes(&dist.client(2)?, &ordinary, "normal.bin", phase.as_bytes()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_seal_requires_every_configured_receiver_on_exact_retry() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
|
||||
let key = format!("{prefix}existing.bin");
|
||||
let output = dist
|
||||
.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(b"existing".to_vec().into())
|
||||
.send()
|
||||
.await?;
|
||||
let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
|
||||
assert_exact_version(&dist.client(3)?, bucket, &key, &version, b"existing").await?;
|
||||
let ordinary = unique_bucket("ordinary");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
|
||||
// The second iteration starts from a fully acknowledged global seal. Its
|
||||
// persisted record cannot replace a fresh acknowledgement from every disk.
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
assert_seal_unconfirmed(&dist, &user, &request).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
}
|
||||
assert_put_denied(&dist.client(3)?, &ordinary, bucket, &format!("{prefix}late.bin")).await?;
|
||||
assert_exact_version(&dist.client(2)?, bucket, &key, &version, b"existing").await?;
|
||||
let listed = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(0)?.list_object_versions().bucket(bucket).prefix(prefix).send(),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(listed.versions().len(), 1, "rejected PUT added a version");
|
||||
assert!(listed.delete_markers().is_empty(), "seal added a delete marker");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_delete_test::physical_snapshot;
|
||||
use super::artifact_repair_test::{RepairObject, finish_heal, start_heal};
|
||||
use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, s3_probe_client, unique_bucket, wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention,
|
||||
ObjectLockRetentionMode,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
const WRITES_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-writes";
|
||||
const INTERNAL_MESSAGE: &str = "We encountered an internal error, please try again.";
|
||||
|
||||
async fn seal_writes(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
|
||||
wait_until(Duration::from_secs(45), || async {
|
||||
let (status, response) = user.request(dist, 0, WRITES_ROUTE, json!({"version":1,"target":target}).to_string()).await?;
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed; retry the same identity") {
|
||||
return Ok(!confirmed);
|
||||
}
|
||||
assert!(confirmed, "offline configured receiver produced a complete receipt: {status} {response}");
|
||||
assert_eq!(status, StatusCode::OK, "combined writes seal: {response}");
|
||||
assert_eq!(serde_json::from_str::<Value>(&response)?, json!({"version":1,"target":target,"ordinary_put_sealed":true,"attempt_repairs_sealed":true,"all_writes_sealed":true}));
|
||||
Ok(true)
|
||||
}, "combined seal acknowledges both domains on every receiver").await
|
||||
}
|
||||
|
||||
async fn assert_put_closed(dist: &DistCluster, bucket: &str, key: &str) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let client = s3_probe_client(&dist.client(2)?);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(b"must-not-publish".to_vec().into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("combined seal admitted a new ordinary PUT");
|
||||
let service = error.as_service_error().ok_or("PUT was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"rejected PUT changed the managed physical namespace"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn lock_metadata_entrypoints(dist: &DistCluster, managed: &Value, object: &RepairObject, ordinary: &str) -> TestResult {
|
||||
let bucket = managed["bucket"].as_str().ok_or("bucket identity")?;
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
for bucket in [bucket, ordinary] {
|
||||
client
|
||||
.put_object_lock_configuration()
|
||||
.bucket(bucket)
|
||||
.object_lock_configuration(
|
||||
ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
for node in 0..4 {
|
||||
let output = match dist.client(node)?.get_object_lock_configuration().bucket(bucket).send().await {
|
||||
Ok(output) => output,
|
||||
Err(error)
|
||||
if error.as_service_error().and_then(ProvideErrorMetadata::code)
|
||||
== Some("ObjectLockConfigurationNotFoundError") =>
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if output
|
||||
.object_lock_configuration()
|
||||
.and_then(|configuration| configuration.object_lock_enabled())
|
||||
!= Some(&ObjectLockEnabled::Enabled)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"actual Object Lock configuration is enabled on all request nodes",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let control = RepairObject::put(dist, ordinary, "lock-control.bin".to_owned()).await?;
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.retain_until_date(aws_sdk_s3::primitives::DateTime::from_secs(chrono::Utc::now().timestamp() + 86400))
|
||||
.build();
|
||||
let hold = ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::On).build();
|
||||
client
|
||||
.put_object_retention()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.retention(retention.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
client
|
||||
.get_object_retention()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.send()
|
||||
.await?
|
||||
.retention()
|
||||
.and_then(|value| value.mode()),
|
||||
Some(&ObjectLockRetentionMode::Governance)
|
||||
);
|
||||
client
|
||||
.put_object_legal_hold()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.legal_hold(hold.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
client
|
||||
.get_object_legal_hold()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.send()
|
||||
.await?
|
||||
.legal_hold()
|
||||
.and_then(|value| value.status()),
|
||||
Some(&ObjectLockLegalHoldStatus::On)
|
||||
);
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let error = client
|
||||
.put_object_retention()
|
||||
.bucket(bucket)
|
||||
.key(&object.key)
|
||||
.version_id(&object.version)
|
||||
.retention(retention)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("managed Retention must reach and fail its unsupported metadata write");
|
||||
let service = error.as_service_error().ok_or("retention was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
|
||||
assert!(physical_snapshot(dist, bucket)? == before, "retention changed managed metadata");
|
||||
let error = client
|
||||
.put_object_legal_hold()
|
||||
.bucket(bucket)
|
||||
.key(&object.key)
|
||||
.version_id(&object.version)
|
||||
.legal_hold(hold)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("managed LegalHold must reach and fail its unsupported metadata write");
|
||||
let service = error.as_service_error().ok_or("legal hold was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some("Io error: artifact bucket does not support this write operation"));
|
||||
assert!(physical_snapshot(dist, bucket)? == before, "legal hold changed managed metadata");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_writes_seal_first_partial_retry_offline_and_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let ordinary = unique_bucket("writes-control");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary).await?;
|
||||
let (fresh, fresh_user) = create_artifact_attempt(&dist).await?;
|
||||
let fresh_bucket = fresh["bucket"].as_str().ok_or("fresh bucket")?;
|
||||
let fresh_prefix = fresh["attempt_prefix"].as_str().ok_or("fresh prefix")?;
|
||||
let first = RepairObject::put(&dist, fresh_bucket, format!("{fresh_prefix}existing.bin")).await?;
|
||||
lock_metadata_entrypoints(&dist, &fresh, &first, &ordinary).await?;
|
||||
seal_writes(&dist, &fresh_user, &fresh, true).await?;
|
||||
seal_writes(&dist, &fresh_user, &fresh, true).await?;
|
||||
assert_put_closed(&dist, fresh_bucket, &format!("{fresh_prefix}closed.bin")).await?;
|
||||
|
||||
let (partial, partial_user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = partial["bucket"].as_str().ok_or("partial bucket")?;
|
||||
let prefix = partial["attempt_prefix"].as_str().ok_or("partial prefix")?;
|
||||
let repaired = RepairObject::put(&dist, bucket, format!("{prefix}repair.bin")).await?;
|
||||
// Enter the combined operation with only its ordinary domain closed.
|
||||
confirm_seal(&dist, &partial_user, &json!({"version":1,"target":partial}).to_string(), &partial).await?;
|
||||
let part = repaired.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), bucket, 3)?;
|
||||
std::fs::remove_file(&part)?;
|
||||
let token = start_heal(&dist, bucket, &repaired.key, false).await?;
|
||||
repaired.assert_repaired(&dist, bucket).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
seal_writes(&dist, &partial_user, &partial, false).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
seal_writes(&dist, &partial_user, &partial, true).await?;
|
||||
}
|
||||
let ordinary_object = RepairObject::put(&dist, &ordinary, "repair-control.bin".to_owned()).await?;
|
||||
for restarted in [false, true] {
|
||||
if restarted {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
seal_writes(&dist, &partial_user, &partial, true).await?;
|
||||
assert_put_closed(&dist, bucket, &format!("{prefix}closed-{restarted}.bin")).await?;
|
||||
if part.exists() {
|
||||
std::fs::remove_file(&part)?;
|
||||
}
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let token = start_heal(&dist, bucket, &repaired.key, false).await?;
|
||||
finish_heal(&dist, &token, Some(&repaired.key)).await?;
|
||||
assert!(!part.exists(), "combined seal admitted a remote repair publication");
|
||||
assert!(physical_snapshot(&dist, bucket)? == before, "rejected repair changed metadata or shards");
|
||||
assert_exact_version(&dist.client(2)?, bucket, &repaired.key, &repaired.version, &repaired.body).await?;
|
||||
let ordinary_part = ordinary_object.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), &ordinary, 3)?;
|
||||
std::fs::remove_file(ordinary_part)?;
|
||||
let token = start_heal(&dist, &ordinary, &ordinary_object.key, false).await?;
|
||||
ordinary_object.assert_repaired(&dist, &ordinary).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -18,6 +18,13 @@
|
||||
//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
|
||||
//! each case starts four real `rustfs` processes.
|
||||
|
||||
mod artifact_delete_test;
|
||||
mod artifact_inventory_test;
|
||||
mod artifact_put_tail_test;
|
||||
mod artifact_repair_tail_test;
|
||||
mod artifact_repair_test;
|
||||
mod artifact_seal_test;
|
||||
mod artifact_writes_test;
|
||||
mod chaos_test;
|
||||
mod concurrency_stability_test;
|
||||
mod concurrent_data_movement_test;
|
||||
|
||||
@@ -1206,7 +1206,9 @@ mod tests {
|
||||
let interval = interval
|
||||
.filter(|interval| *interval != 0)
|
||||
.ok_or("scanner schedule has no positive interval")?;
|
||||
let next_cycle = cycle_end
|
||||
// A new leader may already be sleeping before completing its first cycle.
|
||||
let schedule_start = if cycle_end == 0 { now } else { cycle_end };
|
||||
let next_cycle = schedule_start
|
||||
.checked_add(interval)
|
||||
.ok_or("scanner schedule timestamp overflows")?;
|
||||
let budget = next_cycle
|
||||
@@ -1223,6 +1225,11 @@ mod tests {
|
||||
Ok(Duration::from_secs(259)),
|
||||
);
|
||||
assert_eq!(scanner_cycle_completion_budget(100, Some(1), 110, false), Ok(Duration::from_secs(60)));
|
||||
// A newly elected leader can advertise a sleep before its first completed cycle.
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_budget(0, Some(289), 1_792_913_211, false),
|
||||
Ok(Duration::from_secs(349)),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1243,7 +1250,7 @@ mod tests {
|
||||
previous_cycle_end: u64,
|
||||
) -> Result<u64, Box<dyn Error + Send + Sync>> {
|
||||
let mut deadline = Instant::now() + Duration::from_secs(60);
|
||||
let mut schedule_accounted = false;
|
||||
let mut accounted_schedules = HashSet::new();
|
||||
loop {
|
||||
let mut latest_cycle_end = 0;
|
||||
let mut versions_observed = false;
|
||||
@@ -1290,15 +1297,27 @@ mod tests {
|
||||
return Err("scanner cycle timestamp is in the future".into());
|
||||
}
|
||||
let schedule = &status["cycle_schedule"];
|
||||
if !schedule_accounted && schedule["execution_role"].as_str() == Some("leader") {
|
||||
if schedule["execution_role"].as_str() == Some("leader") {
|
||||
let active = metrics["current_cycle_active"]
|
||||
.as_bool()
|
||||
.ok_or("scanner status is missing its active-cycle flag")?;
|
||||
if active || (cycle_end != 0 && schedule["effective_interval_available"].as_bool() == Some(true)) {
|
||||
let interval = schedule["effective_interval_seconds"].as_u64();
|
||||
let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
|
||||
deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
|
||||
schedule_accounted = true;
|
||||
if active || schedule["effective_interval_available"].as_bool() == Some(true) {
|
||||
let interval = if active {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
schedule["effective_interval_seconds"]
|
||||
.as_u64()
|
||||
.ok_or("scanner schedule is missing its effective interval")?,
|
||||
)
|
||||
};
|
||||
// Reconsider a changed leader or retry plan, but never renew
|
||||
// the deadline on repeated observations of the same sleep.
|
||||
if accounted_schedules.insert((node_index, cycle_end, interval)) {
|
||||
let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
|
||||
let scheduled_deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
|
||||
deadline = deadline.max(scheduled_deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Superseded walks complete all sets while withholding quota authority.
|
||||
|
||||
@@ -541,6 +541,50 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_repair_rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_repair_write_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteMetadataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type InventoryArtifactAttemptStream =
|
||||
Pin<Box<dyn Stream<Item = Result<rustfs_protos::proto_gen::node_service::ArtifactInventoryFrame, Status>> + Send>>;
|
||||
async fn inventory_artifact_attempt(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactInventoryRequest>,
|
||||
) -> Result<Response<Self::InventoryArtifactAttemptStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn seal_artifact_attempt_repairs(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn seal_artifact_attempt_puts(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -15,12 +15,14 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption, Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskError, DiskOption, Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions, new_disk,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
verify_tonic_boot_epoch_response,
|
||||
set_tonic_canonical_body_digest, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
@@ -36,6 +38,11 @@ pub(crate) mod grpc_lock {
|
||||
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod artifact_rpc {
|
||||
pub(crate) use super::{DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
|
||||
}
|
||||
|
||||
/// Signing/transport surface used by the cross-process internode RPC signature
|
||||
/// acceptance tests (backlog#1327). The signing helpers are what let a test mint
|
||||
/// the one legitimately signed request an on-path attacker is assumed to have
|
||||
|
||||
@@ -382,6 +382,43 @@ pub mod config {
|
||||
with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
|
||||
with_server_config_write_lock,
|
||||
};
|
||||
|
||||
/// Observe persisted config bytes and their reader-associated metadata within a byte limit.
|
||||
pub async fn read_config_observation(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
/// Observe bounded config bytes while the caller retains the object's write owner.
|
||||
pub async fn read_config_observation_no_lock(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation_no_lock(api, file, max_bytes, guard).await
|
||||
}
|
||||
|
||||
/// Read config bytes and metadata without treating an empty payload as absent.
|
||||
pub async fn read_config_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
|
||||
}
|
||||
|
||||
pub async fn read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
}
|
||||
|
||||
pub mod storageclass {
|
||||
@@ -431,16 +468,19 @@ pub mod disk {
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
|
||||
ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
is_quota_mutation_fence_path, new_disk, validate_batch_read_version_item_count,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, LegacyFileStream, LegacyFileWrite,
|
||||
LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete, LegacyPathDeleteOperation,
|
||||
LegacyRenameData, LegacyRenameFile, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path,
|
||||
new_disk, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
@@ -635,6 +675,7 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,120 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
|
||||
use crate::config::com::read_config_observation;
|
||||
use crate::object_api::WriteCompletion;
|
||||
|
||||
impl ECStore {
|
||||
/// Permanently retain this managed bucket's versions for historical restores.
|
||||
/// Lifecycle WRITE drains earlier exact deletes. The detached owner keeps
|
||||
/// lifecycle -> metadata transaction locks until every metadata write drains.
|
||||
pub async fn retain_artifact_backups(self: Arc<Self>, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<()> {
|
||||
let bucket = bucket.to_owned();
|
||||
let base_prefix = base_prefix.to_owned();
|
||||
let ctx = Arc::clone(&self.ctx);
|
||||
ctx.run_detached_mutation(async move {
|
||||
require_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact backup retention requires namespace locking"));
|
||||
}
|
||||
let lifecycle = self.acquire_bucket_lifecycle_write_lock(&bucket).await?;
|
||||
let transaction = metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, &bucket).await?;
|
||||
let mut metadata = self.backup_retention_metadata(&bucket, incarnation, &base_prefix).await?;
|
||||
let check = || {
|
||||
if lifecycle.is_lock_lost() || transaction.is_lock_lost() {
|
||||
Err(Error::other("artifact backup retention fence was lost"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
check()?;
|
||||
// Even an idempotent retry rewrites with TailDrained: an earlier
|
||||
// interrupted call may have reached only some metadata replicas.
|
||||
metadata.artifact_backup_retained = true;
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
opts.add_namespace_lock_guard(&transaction);
|
||||
metadata.save_with_store_options(self.clone(), &opts).await?;
|
||||
check()?;
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
sys.set_validated(metadata).await?;
|
||||
check()
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact backup retention owner failed"))?
|
||||
}
|
||||
|
||||
/// Observe the immutable bucket identity and retention in one locked read.
|
||||
pub async fn inspect_artifact_backup_retention(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<bool> {
|
||||
require_put_seal_topology(self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact backup retention requires namespace locking"));
|
||||
}
|
||||
let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let metadata = self.backup_retention_metadata(bucket, incarnation, base_prefix).await?;
|
||||
if lifecycle.is_lock_lost() {
|
||||
return Err(Error::other("artifact backup retention observation fence was lost"));
|
||||
}
|
||||
Ok(metadata.artifact_backup_retained)
|
||||
}
|
||||
|
||||
async fn backup_retention_metadata(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<BucketMetadata> {
|
||||
if incarnation.is_nil() || !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(base_prefix) {
|
||||
return Err(Error::other("artifact backup retention identity is invalid"));
|
||||
}
|
||||
let mut metadata = self.read_backup_retention_metadata(bucket).await?;
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let (sidecar, _) =
|
||||
read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
|
||||
metadata.parse_all_configs()?;
|
||||
if metadata.bucket_incarnation_id != incarnation
|
||||
|| Uuid::from_slice(&sidecar).ok() != Some(incarnation)
|
||||
|| metadata.artifact_base_prefix.as_deref() != Some(base_prefix)
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| !metadata.versioning_config.as_ref().is_some_and(|versioning| {
|
||||
versioning.enabled()
|
||||
&& !versioning.exclude_folders.unwrap_or(false)
|
||||
&& versioning.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
})
|
||||
{
|
||||
return Err(Error::other("artifact backup retention identity mismatch"));
|
||||
}
|
||||
metadata.bucket_incarnation_sidecar = true;
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
/// Caller must hold the bucket lifecycle fence. An unreadable or missing
|
||||
/// record cannot prove that historical backup protection was never enabled.
|
||||
pub(crate) async fn require_no_artifact_backup_retention(&self, bucket: &str) -> Result<()> {
|
||||
if self.read_backup_retention_metadata(bucket).await?.artifact_backup_retained {
|
||||
return Err(Error::other("artifact versions are retained for backups"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_backup_retention_metadata(&self, bucket: &str) -> Result<BucketMetadata> {
|
||||
if bucket.is_empty()
|
||||
|| super::super::utils::has_bad_path_component(bucket)
|
||||
|| super::super::utils::is_meta_bucketname(bucket)
|
||||
{
|
||||
return Err(Error::other("artifact backup retention bucket is invalid"));
|
||||
}
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if store.id != self.id {
|
||||
return Err(Error::other("artifact backup retention belongs to another store"));
|
||||
}
|
||||
let (bytes, _) = read_config_observation(store, &BucketMetadata::new(bucket).save_file_path(), 64 * 1024 * 1024).await?;
|
||||
let metadata = BucketMetadata::decode_file(&bytes)?;
|
||||
if metadata.name != bucket {
|
||||
return Err(Error::other("artifact backup retention bucket mismatch"));
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
|
||||
/// In-process authority for exactly one drained managed version. No caller can
|
||||
/// construct this capability or serialize it into a disk/RPC request.
|
||||
#[doc(hidden)]
|
||||
pub struct ArtifactDeleteTarget {
|
||||
store: Uuid,
|
||||
record: ArtifactVersionRetirementRecord,
|
||||
lifecycle: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
attempt: WriteCommitGuard,
|
||||
repair: WriteCommitGuard,
|
||||
}
|
||||
|
||||
impl ArtifactDeleteTarget {
|
||||
fn check_live(&self) -> Result<()> {
|
||||
if self.lifecycle.is_lock_lost() || self.attempt.is_lock_lost() || self.repair.is_lock_lost() {
|
||||
return Err(Error::other("artifact exact delete owned fence was lost"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn lifecycle(&self) -> Arc<rustfs_lock::NamespaceLockGuard> {
|
||||
Arc::clone(&self.lifecycle)
|
||||
}
|
||||
|
||||
pub(crate) fn validate(&self, store: &ECStore, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
if self.store != store.id
|
||||
|| bucket != self.record.bucket
|
||||
|| rustfs_utils::path::encode_dir_object(object) != self.record.object
|
||||
|| opts.expected_bucket_incarnation_id != Some(self.record.incarnation_id)
|
||||
|| opts.version_id.as_deref() != Some(self.record.version_id.to_string().as_str())
|
||||
|| !opts.versioned
|
||||
|| opts.version_suspended
|
||||
|| opts.delete_prefix
|
||||
|| opts.delete_marker
|
||||
|| opts.data_movement
|
||||
|| opts.lifecycle_delete_all.is_some()
|
||||
|| self.lifecycle.is_lock_lost()
|
||||
|| self.attempt.is_lock_lost()
|
||||
|| self.repair.is_lock_lost()
|
||||
{
|
||||
return Err(Error::other("artifact exact delete target or owned fence mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Read-only capability observation; each mutation revalidates the topology.
|
||||
pub async fn supports_artifact_exact_delete(&self) -> bool {
|
||||
require_local_put_seal_topology(self).await.is_ok() && !self.ctx.lock_manager().is_disabled()
|
||||
}
|
||||
|
||||
/// Delete an exact version in a fully sealed attempt. This authenticates the
|
||||
/// storage identity and drains writers; the caller still owns GC eligibility
|
||||
/// and cleanup IAM. Distributed receiver deletion is not yet supported.
|
||||
pub async fn delete_artifact_object_version(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
expected_incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
) -> Result<()> {
|
||||
require_local_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact exact deletion requires namespace locking"));
|
||||
}
|
||||
let lifecycle = Arc::new(self.acquire_bucket_lifecycle_read_lock(bucket).await?);
|
||||
self.require_no_artifact_backup_retention(bucket).await?;
|
||||
let put = self
|
||||
.inventory_artifact_record(bucket, expected_incarnation, attempt_prefix)
|
||||
.await?;
|
||||
let repair = ArtifactRepairSealRecord::new(&put);
|
||||
let record = ArtifactVersionRetirementRecord::new(&put, object, version)?;
|
||||
let object = object.to_owned();
|
||||
let store = Arc::clone(&self);
|
||||
// Lifecycle -> attempt -> repair seal -> metadata transaction -> object.
|
||||
// The spawned owner, and the disk deletion tail's typed capability, keep
|
||||
// these locks after cancellation of the HTTP caller.
|
||||
with_config_object_write_lock(self, put.path(), move |attempt| async move {
|
||||
if !put.exists(Arc::clone(&store), &attempt).await? {
|
||||
return Err(Error::other("artifact exact deletion requires sealed PUTs"));
|
||||
}
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
|
||||
let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
|
||||
if !repair.exists(Arc::clone(&store), &repair_owner).await? {
|
||||
return Err(Error::other("artifact exact deletion requires sealed repairs"));
|
||||
}
|
||||
let receivers = store.capture_artifact_receivers().await?;
|
||||
store
|
||||
.close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Put, &receivers)
|
||||
.await?;
|
||||
store
|
||||
.close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Repair, &receivers)
|
||||
.await?;
|
||||
store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
|
||||
let target = Arc::new(ArtifactDeleteTarget {
|
||||
store: store.id,
|
||||
record,
|
||||
lifecycle,
|
||||
attempt,
|
||||
repair: repair_owner,
|
||||
});
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.to_string()),
|
||||
expected_bucket_incarnation_id: Some(expected_incarnation),
|
||||
artifact_delete_target: Some(Arc::clone(&target)),
|
||||
delete_mutation_owner: Some(target.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_write_commit_guard(&target.attempt);
|
||||
opts.add_write_commit_guard(&target.repair);
|
||||
// These owners fence seal records, not the object. Ordinary DELETE
|
||||
// must still acquire and retain its exact object WRITE owner.
|
||||
opts.no_lock = false;
|
||||
target.check_live()?;
|
||||
// Preserve the normal exact DELETE's Object Lock, quorum, journal,
|
||||
// cache and capacity behavior. Its success is logical deletion, not
|
||||
// a claim that every offline shard has been physically erased.
|
||||
match store.delete_object(&put.bucket, &object, opts).await {
|
||||
Ok(info) => {
|
||||
// Match the S3 request layer's committed-generation quota
|
||||
// delta. A strict absence retry must not subtract twice.
|
||||
if !info.name.is_empty()
|
||||
&& !info.delete_marker
|
||||
&& let Ok(size) = crate::data_usage::quota_object_size(&info)
|
||||
{
|
||||
crate::data_usage::record_bucket_object_delete_memory(&put.bucket, size, false).await;
|
||||
}
|
||||
}
|
||||
Err(Error::VersionNotFound(bucket, object, version))
|
||||
if bucket == target.record.bucket
|
||||
&& rustfs_utils::path::encode_dir_object(&object) == target.record.object
|
||||
&& version == target.record.version_id.to_string() => {}
|
||||
Err(Error::ObjectNotFound(bucket, object))
|
||||
if bucket == target.record.bucket
|
||||
&& rustfs_utils::path::encode_dir_object(&object) == target.record.object => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
|
||||
store
|
||||
.inventory_artifact_record(&put.bucket, expected_incarnation, &put.attempt_prefix)
|
||||
.await?;
|
||||
target.check_live()
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion, Disk};
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase;
|
||||
use tokio::{sync::mpsc, time::Instant};
|
||||
|
||||
/// Physical observations retain their member. Replicas are not extra business
|
||||
/// versions and this stream makes no quorum-readability or GC claim.
|
||||
#[derive(Debug)]
|
||||
pub enum ArtifactAttemptInventoryFrame {
|
||||
Observations {
|
||||
endpoint: String,
|
||||
disk_id: Uuid,
|
||||
versions: Vec<ArtifactInventoryVersion>,
|
||||
},
|
||||
Complete {
|
||||
members: usize,
|
||||
observations: u64,
|
||||
},
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Inventory one fully sealed attempt across every configured receiver.
|
||||
/// Earlier output is staging only; only the final frame plus Ok is complete.
|
||||
pub async fn inventory_artifact_attempt(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
prefix: &str,
|
||||
deadline: Instant,
|
||||
output: mpsc::Sender<ArtifactAttemptInventoryFrame>,
|
||||
) -> Result<()> {
|
||||
tokio::time::timeout_at(deadline, Box::pin(async move {
|
||||
require_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() { return Err(Error::other("artifact inventory requires namespace locking")); }
|
||||
let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let put = self.inventory_artifact_record(bucket, incarnation, prefix).await?;
|
||||
let repair = ArtifactRepairSealRecord::new(&put);
|
||||
// Lifecycle -> attempt -> repair record. Receivers do not reenter
|
||||
// distributed locks; actual two-domain disk owners guard their reads.
|
||||
let store = self.clone();
|
||||
let bucket = bucket.to_owned();
|
||||
let prefix = prefix.to_owned();
|
||||
with_config_object_write_lock(self, put.path(), move |attempt| async move {
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
if !put.exists(store.clone(), &attempt).await? { return Err(Error::other("artifact inventory requires PUT seal record")); }
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
|
||||
let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
|
||||
if !repair.exists(store.clone(), &repair_owner).await? { return Err(Error::other("artifact inventory requires repair seal record")); }
|
||||
let receivers = store.capture_artifact_receivers().await?;
|
||||
let mut observations = 0u64;
|
||||
for (endpoint, (disk, expected)) in &receivers {
|
||||
if lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
|
||||
return Err(Error::other("artifact inventory fence was lost"));
|
||||
}
|
||||
let (sender, mut frames) = mpsc::channel(2);
|
||||
let scan = async {
|
||||
match disk.as_ref() {
|
||||
Disk::Local(local) => local.get_disk().inventory_artifact_attempt(*expected, &bucket, &prefix, put.identity(), deadline, sender).await,
|
||||
Disk::Remote(remote) => remote.inventory_artifact_attempt(*expected, (&bucket, incarnation), &prefix, ArtifactInventoryPhase::ArtifactInventoryScan, deadline, sender).await,
|
||||
}
|
||||
};
|
||||
tokio::pin!(scan);
|
||||
let mut sequence = 0;
|
||||
let mut count = 0u64;
|
||||
let mut terminal = false;
|
||||
let mut ended = false;
|
||||
let mut done = false;
|
||||
while !ended || !done {
|
||||
tokio::select! {
|
||||
result = &mut scan, if !done => { result.map_err(Error::other)?; done = true; },
|
||||
frame = frames.recv(), if !ended => match frame {
|
||||
Some(ArtifactInventoryFrame::Entries { sequence: actual, versions }) if !terminal && actual == sequence => {
|
||||
count = count.checked_add(u64::try_from(versions.len()).map_err(Error::other)?).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
|
||||
output.send(ArtifactAttemptInventoryFrame::Observations { endpoint: endpoint.clone(), disk_id: *expected, versions }).await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
sequence += 1;
|
||||
}
|
||||
Some(ArtifactInventoryFrame::Complete { disk_id, incarnation: actual, attempt_prefix, entries })
|
||||
if !terminal && disk_id == *expected && actual == incarnation && attempt_prefix == prefix && entries == count => terminal = true,
|
||||
Some(_) => return Err(Error::other("artifact inventory member stream is invalid")),
|
||||
None => ended = true,
|
||||
},
|
||||
}
|
||||
}
|
||||
if !terminal { return Err(Error::other("artifact inventory member is incomplete")); }
|
||||
observations = observations.checked_add(count).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
|
||||
}
|
||||
store.verify_inventory_members(&put, &receivers, deadline).await?;
|
||||
let current = store.artifact_inventory_receiver_identity(&bucket, incarnation, &prefix).await?;
|
||||
if current.receiver_key != put.identity().receiver_key || lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
|
||||
return Err(Error::other("artifact inventory authority changed"));
|
||||
}
|
||||
output.send(ArtifactAttemptInventoryFrame::Complete { members: receivers.len(), observations }).await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
Ok(())
|
||||
}).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
|
||||
}).await?
|
||||
})).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
|
||||
}
|
||||
|
||||
pub(super) async fn inventory_artifact_record(&self, bucket: &str, expected: Uuid, prefix: &str) -> Result<PutSealRecord> {
|
||||
use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
|
||||
use crate::config::com::read_config_observation;
|
||||
if expected.is_nil()
|
||||
|| crate::bucket::utils::has_bad_path_component(bucket)
|
||||
|| bucket.is_empty()
|
||||
|| crate::bucket::utils::is_meta_bucketname(bucket)
|
||||
{
|
||||
return Err(Error::other("artifact inventory authority is invalid"));
|
||||
}
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if store.id != self.id {
|
||||
return Err(Error::other("artifact inventory authority belongs to another store"));
|
||||
}
|
||||
// Do not enter metadata_sys's lazy bucket READ/migration path: the
|
||||
// coordinator already owns lifecycle READ and a writer may be queued.
|
||||
// Observation reads bypass cache and suppress physical read repair.
|
||||
let path = BucketMetadata::new(bucket).save_file_path();
|
||||
let (bytes, _) = read_config_observation(store.clone(), &path, 64 * 1024 * 1024).await?;
|
||||
let mut metadata = BucketMetadata::decode_file(&bytes)?;
|
||||
let (sidecar, _) =
|
||||
read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
|
||||
if metadata.name != bucket
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| metadata.bucket_incarnation_id != expected
|
||||
|| Uuid::from_slice(&sidecar).ok() != Some(expected)
|
||||
{
|
||||
return Err(Error::other("artifact inventory incarnation mismatch"));
|
||||
}
|
||||
metadata.parse_all_configs()?;
|
||||
if !metadata.versioning_config.as_ref().is_some_and(|config| {
|
||||
config.enabled()
|
||||
&& !config.exclude_folders.unwrap_or(false)
|
||||
&& config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
}) {
|
||||
return Err(Error::other("artifact inventory requires managed versioning"));
|
||||
}
|
||||
PutSealRecord::new(&metadata, prefix)
|
||||
}
|
||||
|
||||
pub(crate) async fn artifact_inventory_receiver_identity(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: Uuid,
|
||||
prefix: &str,
|
||||
) -> Result<ArtifactPutIdentity> {
|
||||
Ok(self.inventory_artifact_record(bucket, expected, prefix).await?.identity())
|
||||
}
|
||||
|
||||
async fn verify_inventory_members(
|
||||
&self,
|
||||
record: &PutSealRecord,
|
||||
captured: &ArtifactReceiverSnapshot,
|
||||
deadline: Instant,
|
||||
) -> Result<()> {
|
||||
for final_pass in [false, true] {
|
||||
let current = self.artifact_receiver_inventory().await?;
|
||||
if current.len() != captured.len()
|
||||
|| current
|
||||
.iter()
|
||||
.any(|(endpoint, disk)| !captured.get(endpoint).is_some_and(|(old, _)| Arc::ptr_eq(old, disk)))
|
||||
{
|
||||
return Err(Error::other("artifact inventory membership changed"));
|
||||
}
|
||||
if final_pass {
|
||||
for (disk, expected) in captured.values() {
|
||||
if disk.cached_disk_id().await != Some(*expected) {
|
||||
return Err(Error::other("artifact inventory expected identity changed"));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
for (disk, expected) in captured.values() {
|
||||
if disk.cached_disk_id().await != Some(*expected) {
|
||||
return Err(Error::other("artifact inventory expected identity changed"));
|
||||
}
|
||||
match disk.as_ref() {
|
||||
Disk::Local(local) => {
|
||||
let _verification = local
|
||||
.get_disk()
|
||||
.verify_artifact_inventory_closed(*expected, record.identity(), deadline)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
}
|
||||
Disk::Remote(remote) => {
|
||||
let (output, _unused) = mpsc::channel(1);
|
||||
remote
|
||||
.inventory_artifact_attempt(
|
||||
*expected,
|
||||
(&record.bucket, record.incarnation_id),
|
||||
&record.attempt_prefix,
|
||||
ArtifactInventoryPhase::ArtifactInventoryVerify,
|
||||
deadline,
|
||||
output,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
|
||||
/// Shares only reads started after a caller arrives; retains no idle results.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct OrdinaryClassificationRegistry {
|
||||
gates: Mutex<HashMap<String, Weak<Gate>>>,
|
||||
#[cfg(test)]
|
||||
after_unregister: Option<std::sync::Barrier>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Gate {
|
||||
started_generation: AtomicU64,
|
||||
completed: tokio::sync::Mutex<Option<CompletedRead>>,
|
||||
}
|
||||
|
||||
struct CompletedRead {
|
||||
generation: u64,
|
||||
allowed: bool,
|
||||
}
|
||||
|
||||
struct Registration<'a> {
|
||||
registry: &'a OrdinaryClassificationRegistry,
|
||||
bucket: &'a str,
|
||||
gate: Option<Arc<Gate>>,
|
||||
observed_generation: u64,
|
||||
}
|
||||
|
||||
impl OrdinaryClassificationRegistry {
|
||||
pub(crate) async fn classify(&self, bucket: &str, read: impl Future<Output = bool>) -> bool {
|
||||
self.register(bucket).classify(read).await
|
||||
}
|
||||
|
||||
fn register<'a>(&'a self, bucket: &'a str) -> Registration<'a> {
|
||||
// Callers requiring physical admission must register after acquiring it.
|
||||
// Release the registry lock before awaiting the gate, then acquire the
|
||||
// existing metadata locks.
|
||||
let mut gates = self.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let gate = gates.get(bucket).and_then(Weak::upgrade).unwrap_or_else(|| {
|
||||
let gate = Arc::new(Gate::default());
|
||||
gates.insert(bucket.to_owned(), Arc::downgrade(&gate));
|
||||
gate
|
||||
});
|
||||
let observed_generation = gate.started_generation.load(Ordering::Acquire);
|
||||
Registration {
|
||||
registry: self,
|
||||
bucket,
|
||||
gate: Some(gate),
|
||||
observed_generation,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Registration<'_> {
|
||||
async fn classify(self, read: impl Future<Output = bool>) -> bool {
|
||||
let Some(gate) = self.gate.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
let mut completed = gate.completed.lock().await;
|
||||
// Only a read started after this caller registered can authorize it.
|
||||
// Arrivals during a read therefore wait for the next fresh read.
|
||||
if let Some(result) = completed
|
||||
.as_ref()
|
||||
.filter(|result| result.generation > self.observed_generation)
|
||||
{
|
||||
return result.allowed;
|
||||
}
|
||||
let Ok(previous) = gate
|
||||
.started_generation
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
// Keep the gate through the original bounded metadata read. Cancellation
|
||||
// releases it without publishing a result; another caller can read again.
|
||||
let allowed = read.await;
|
||||
*completed = Some(CompletedRead {
|
||||
generation: previous + 1,
|
||||
allowed,
|
||||
});
|
||||
allowed
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Registration<'_> {
|
||||
fn drop(&mut self) {
|
||||
let Some(gate) = self.gate.take() else {
|
||||
return;
|
||||
};
|
||||
let mut gates = self.registry.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Strong references are acquired and released under this registry lock,
|
||||
// so concurrent final callers cannot both miss the last reference.
|
||||
if Arc::strong_count(&gate) == 1
|
||||
&& gates
|
||||
.get(self.bucket)
|
||||
.is_some_and(|registered| registered.ptr_eq(&Arc::downgrade(&gate)))
|
||||
{
|
||||
gates.remove(self.bucket);
|
||||
}
|
||||
drop(gate);
|
||||
drop(gates);
|
||||
#[cfg(test)]
|
||||
if let Some(barrier) = &self.registry.after_unregister {
|
||||
barrier.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures::poll;
|
||||
use std::task::Poll;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
fn assert_empty(registry: &OrdinaryClassificationRegistry) {
|
||||
assert!(registry.gates.lock().expect("classification registry").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_arrivals_share_only_the_next_fail_closed_read() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
let (release, waiting) = oneshot::channel();
|
||||
let mut first = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
waiting.await.expect("release first read");
|
||||
true
|
||||
}));
|
||||
assert!(poll!(first.as_mut()).is_pending());
|
||||
let mut followers = (0..16)
|
||||
.map(|_| {
|
||||
Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
false
|
||||
}))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for follower in &mut followers {
|
||||
assert!(poll!(follower.as_mut()).is_pending());
|
||||
}
|
||||
release.send(()).expect("first reader still alive");
|
||||
assert_eq!(poll!(first.as_mut()), Poll::Ready(true));
|
||||
for follower in &mut followers {
|
||||
assert_eq!(poll!(follower.as_mut()), Poll::Ready(false));
|
||||
}
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 2);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successive_waves_always_read_again() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
for expected in [true, false, true] {
|
||||
assert_eq!(
|
||||
registry
|
||||
.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
expected
|
||||
})
|
||||
.await,
|
||||
expected
|
||||
);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_leader_does_not_publish_or_strand_waiters() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
let old_registration = registry.register("bucket");
|
||||
assert!(
|
||||
registry
|
||||
.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
true
|
||||
})
|
||||
.await
|
||||
);
|
||||
let mut leader = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
std::future::pending::<bool>().await
|
||||
}));
|
||||
assert!(poll!(leader.as_mut()).is_pending());
|
||||
let mut old_waiter =
|
||||
Box::pin(old_registration.classify(async { panic!("old waiter must reuse the completed post-registration read") }));
|
||||
assert!(poll!(old_waiter.as_mut()).is_pending());
|
||||
let mut late_waiter = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
false
|
||||
}));
|
||||
assert!(poll!(late_waiter.as_mut()).is_pending());
|
||||
drop(leader);
|
||||
assert_eq!(poll!(old_waiter.as_mut()), Poll::Ready(true));
|
||||
assert_eq!(poll!(late_waiter.as_mut()), Poll::Ready(false));
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 3);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_follower_does_not_retain_the_gate() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let (release, waiting) = oneshot::channel();
|
||||
let mut leader = Box::pin(registry.classify("bucket", async {
|
||||
waiting.await.expect("release leader");
|
||||
true
|
||||
}));
|
||||
assert!(poll!(leader.as_mut()).is_pending());
|
||||
let mut follower = Box::pin(registry.classify("bucket", async { panic!("canceled follower must not read") }));
|
||||
assert!(poll!(follower.as_mut()).is_pending());
|
||||
drop(follower);
|
||||
assert_eq!(registry.gates.lock().expect("classification registry").len(), 1);
|
||||
release.send(()).expect("leader still alive");
|
||||
assert_eq!(poll!(leader.as_mut()), Poll::Ready(true));
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_final_registrations_remove_the_gate() {
|
||||
let registry = OrdinaryClassificationRegistry {
|
||||
after_unregister: Some(std::sync::Barrier::new(2)),
|
||||
..Default::default()
|
||||
};
|
||||
let first = registry.register("bucket");
|
||||
let second = registry.register("bucket");
|
||||
// Both destructors reach the unlocked barrier before either returns.
|
||||
// Moving the Arc release after this barrier leaves a stale registry key.
|
||||
std::thread::scope(|scope| {
|
||||
scope.spawn(move || drop(first));
|
||||
scope.spawn(move || drop(second));
|
||||
});
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buckets_and_stores_do_not_share_gates() {
|
||||
let first_store = OrdinaryClassificationRegistry::default();
|
||||
let second_store = OrdinaryClassificationRegistry::default();
|
||||
let mut blocked = Box::pin(first_store.classify("first", std::future::pending()));
|
||||
assert!(poll!(blocked.as_mut()).is_pending());
|
||||
assert_eq!(poll!(Box::pin(first_store.classify("second", async { true }))), Poll::Ready(true));
|
||||
assert_eq!(poll!(Box::pin(second_store.classify("first", async { false }))), Poll::Ready(false));
|
||||
assert_empty(&second_store);
|
||||
drop(blocked);
|
||||
assert_empty(&first_store);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generation_exhaustion_fails_closed_without_reading() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let registration = registry.register("bucket");
|
||||
let gate = registration.gate.as_ref().expect("registered gate");
|
||||
gate.started_generation.store(u64::MAX, Ordering::Release);
|
||||
*gate.completed.lock().await = Some(CompletedRead {
|
||||
generation: u64::MAX,
|
||||
allowed: true,
|
||||
});
|
||||
assert!(
|
||||
!registry
|
||||
.classify("bucket", async { panic!("exhausted generation must deny without reading") })
|
||||
.await
|
||||
);
|
||||
assert_eq!(gate.started_generation.load(Ordering::Acquire), u64::MAX);
|
||||
drop(registration);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
}
|
||||
@@ -890,6 +890,7 @@ pub(crate) async fn cleanup_free_version_exact(
|
||||
let bucket_guard = api
|
||||
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
|
||||
.await
|
||||
.map(Arc::new)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
|
||||
let local_object = encode_dir_object(&oi.name);
|
||||
@@ -958,7 +959,7 @@ pub(crate) async fn cleanup_free_version_exact(
|
||||
};
|
||||
delete_request.set_tier_free_version();
|
||||
if let Err(err) = set
|
||||
.delete_object_version(&oi.bucket, &local_object, &delete_request, false)
|
||||
.delete_object_version_with_lifecycle_owner(&oi.bucket, &local_object, &delete_request, Arc::clone(&bucket_guard))
|
||||
.await
|
||||
&& first_error.is_none()
|
||||
{
|
||||
@@ -8496,6 +8497,104 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn artifact_delete_entrypoints_free_version_cleaner_completes_with_queued_lifecycle_writer() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("free-version-queued-writer-{}", Uuid::new_v4());
|
||||
let object = "free-version";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let (backend, identity) = register_recovery_mock_tier(&ecstore).await;
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity)).await;
|
||||
let oi = list_tier_free_versions(Arc::clone(&ecstore), 100, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("list real persisted free version")
|
||||
.items
|
||||
.into_iter()
|
||||
.find(|oi| oi.bucket == bucket)
|
||||
.expect("free-version task must address this fixture");
|
||||
assert!(oi.transitioned_object.free_version);
|
||||
backend
|
||||
.set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
|
||||
.await;
|
||||
let remote = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("remote fixture lease");
|
||||
remote
|
||||
.put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
|
||||
.await
|
||||
.expect("seed exact remote object");
|
||||
drop(remote);
|
||||
|
||||
let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
|
||||
let cleanup_barrier = Arc::clone(&barrier);
|
||||
let cleanup_store = Arc::clone(&ecstore);
|
||||
let cleanup_oi = oi.clone();
|
||||
let mut cleanup = tokio::spawn(async move {
|
||||
super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
|
||||
.scope(cleanup_barrier, async move {
|
||||
super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(10), barrier.arrived.notified())
|
||||
.await
|
||||
.expect("cleaner owns lifecycle READ and has completed remote DELETE");
|
||||
assert!(!backend.contains(&oi.transitioned_object.name).await);
|
||||
for disk in &disk_paths {
|
||||
assert!(
|
||||
disk.join(&bucket).join(object).join(STORAGE_FORMAT_FILE).is_file(),
|
||||
"local exact cleanup must still be pending at the boundary"
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve first so the pending poll is the actual local lock wait.
|
||||
// A later READ would normally be admitted alongside the cleaner's READ;
|
||||
// its pending state independently proves this WRITE is now queued.
|
||||
let lifecycle = ecstore
|
||||
.new_ns_lock(&bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||
.await
|
||||
.expect("lifecycle namespace");
|
||||
let mut writer = Box::pin(lifecycle.get_write_lock(StdDuration::from_secs(30)));
|
||||
let writer_queued = futures::poll!(&mut writer).is_pending();
|
||||
let mut later_reader = Box::pin(lifecycle.get_read_lock(StdDuration::from_secs(30)));
|
||||
let later_reader_blocked = futures::poll!(&mut later_reader).is_pending();
|
||||
drop(later_reader);
|
||||
barrier.release.notify_one();
|
||||
assert!(writer_queued, "cleaner must retain the real lifecycle READ owner");
|
||||
assert!(later_reader_blocked, "queued lifecycle WRITE must block a later READ");
|
||||
|
||||
// Keep the same WRITE future queued, without cancelling/retrying it.
|
||||
// Re-entering READ here would deadlock behind this waiting writer.
|
||||
let finished = tokio::time::timeout(StdDuration::from_secs(10), &mut cleanup).await;
|
||||
if finished.is_err() {
|
||||
cleanup.abort();
|
||||
drop(writer);
|
||||
let _ = tokio::time::timeout(StdDuration::from_secs(5), &mut cleanup).await;
|
||||
panic!("exact local free-version cleanup re-entered lifecycle READ behind the queued writer");
|
||||
}
|
||||
assert!(
|
||||
finished
|
||||
.expect("bounded cleaner completion")
|
||||
.expect("cleaner task")
|
||||
.expect("exact local cleanup")
|
||||
);
|
||||
let writer = tokio::time::timeout(StdDuration::from_secs(10), writer)
|
||||
.await
|
||||
.expect("queued lifecycle WRITE proceeds after real cleanup drains")
|
||||
.expect("lifecycle WRITE acquired");
|
||||
for disk in &disk_paths {
|
||||
assert!(!disk.join(&bucket).join(object).exists(), "free version remains on a real local disk");
|
||||
}
|
||||
assert_eq!(backend.remove_count().await, 1, "cleaner must delete the exact remote tuple once");
|
||||
drop(writer);
|
||||
ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("remove empty fixture bucket");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
|
||||
@@ -437,6 +437,8 @@ pub struct BucketMetadata {
|
||||
pub created: OffsetDateTime,
|
||||
pub lock_enabled: bool, // While marked as unused, it may need to be retained
|
||||
pub bucket_incarnation_id: Uuid,
|
||||
pub artifact_base_prefix: Option<String>,
|
||||
pub artifact_backup_retained: bool,
|
||||
pub(crate) bucket_incarnation_sidecar: bool,
|
||||
pub(crate) bucket_creation_committed: bool,
|
||||
pub policy_config_json: Vec<u8>,
|
||||
@@ -512,6 +514,8 @@ impl Default for BucketMetadata {
|
||||
created: OffsetDateTime::UNIX_EPOCH,
|
||||
lock_enabled: Default::default(),
|
||||
bucket_incarnation_id: Uuid::nil(),
|
||||
artifact_base_prefix: None,
|
||||
artifact_backup_retained: false,
|
||||
bucket_incarnation_sidecar: false,
|
||||
bucket_creation_committed: false,
|
||||
policy_config_json: Default::default(),
|
||||
@@ -607,7 +611,7 @@ impl BucketMetadata {
|
||||
/// legacy migration path, which mints the sidecar first; it must not be
|
||||
/// rewritten by this migration.
|
||||
pub(crate) fn needs_bucket_creation_commit(&self) -> bool {
|
||||
self.lock_enabled
|
||||
(self.lock_enabled || self.artifact_base_prefix.is_some())
|
||||
&& !self.bucket_creation_committed
|
||||
&& self.bucket_incarnation_sidecar
|
||||
&& !self.bucket_incarnation_id.is_nil()
|
||||
@@ -704,6 +708,22 @@ impl BucketMetadata {
|
||||
self.bucket_incarnation_id =
|
||||
Uuid::from_slice(&bytes).map_err(|err| Error::other(format!("invalid BucketIncarnationID: {err}")))?;
|
||||
}
|
||||
"ArtifactBackupRetained" => {
|
||||
if self.artifact_backup_retained || !rmp::decode::read_bool(rd)? {
|
||||
return Err(Error::other("invalid artifact backup retention marker"));
|
||||
}
|
||||
self.artifact_backup_retained = true;
|
||||
}
|
||||
"ArtifactBasePrefix" => {
|
||||
if self.artifact_base_prefix.is_some() {
|
||||
return Err(Error::other("duplicate artifact namespace"));
|
||||
}
|
||||
let prefix = read_msgp_str(rd)?;
|
||||
if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(&prefix) {
|
||||
return Err(Error::other("invalid artifact namespace"));
|
||||
}
|
||||
self.artifact_base_prefix = Some(prefix);
|
||||
}
|
||||
"PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
|
||||
"NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
|
||||
"LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
|
||||
@@ -766,7 +786,7 @@ impl BucketMetadata {
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (22)
|
||||
let map_len: u32 = 47;
|
||||
let map_len: u32 = 47 + u32::from(self.artifact_base_prefix.is_some()) + u32::from(self.artifact_backup_retained);
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -780,6 +800,15 @@ impl BucketMetadata {
|
||||
rmp::encode::write_bool(wr, self.lock_enabled)?;
|
||||
|
||||
write_bin_field(wr, "BucketIncarnationID", self.bucket_incarnation_id.as_bytes())?;
|
||||
if let Some(prefix) = &self.artifact_base_prefix {
|
||||
rmp::encode::write_str(wr, "ArtifactBasePrefix")?;
|
||||
rmp::encode::write_str(wr, prefix)?;
|
||||
}
|
||||
|
||||
if self.artifact_backup_retained {
|
||||
rmp::encode::write_str(wr, "ArtifactBackupRetained")?;
|
||||
rmp::encode::write_bool(wr, true)?;
|
||||
}
|
||||
|
||||
write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?;
|
||||
write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?;
|
||||
@@ -880,13 +909,28 @@ impl BucketMetadata {
|
||||
}
|
||||
|
||||
match version {
|
||||
BUCKET_METADATA_VERSION => {}
|
||||
BUCKET_METADATA_VERSION | 2 => {}
|
||||
_ => return Err(Error::other("read_bucket_metadata: version invalid")),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Version two is reserved for permanent artifact backup retention. Older
|
||||
/// readers reject its header instead of skipping the protection field.
|
||||
pub(crate) fn decode_file(bytes: &[u8]) -> Result<Self> {
|
||||
Self::check_header(bytes)?;
|
||||
let metadata = Self::unmarshal(&bytes[4..])?;
|
||||
let retained_header = LittleEndian::read_u16(&bytes[2..4]) == 2;
|
||||
if retained_header != metadata.artifact_backup_retained {
|
||||
return Err(Error::other("artifact backup retention header mismatch"));
|
||||
}
|
||||
if metadata.artifact_backup_retained {
|
||||
metadata.validate_artifact_configuration()?;
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub(crate) fn default_timestamps(&mut self) {
|
||||
if self.policy_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.policy_config_updated_at = self.created
|
||||
@@ -1025,6 +1069,9 @@ impl BucketMetadata {
|
||||
}
|
||||
Some(config)
|
||||
};
|
||||
if self.artifact_base_prefix.is_some() && !config.as_ref().is_some_and(artifact_versioning_enabled) {
|
||||
return Err(Error::other("artifact bucket versioning must remain enabled"));
|
||||
}
|
||||
self.versioning_config_xml = data;
|
||||
self.versioning_config = config;
|
||||
self.versioning_config_updated_at = updated;
|
||||
@@ -1127,13 +1174,34 @@ impl BucketMetadata {
|
||||
&mut self,
|
||||
store: std::sync::Arc<crate::store::ECStore>,
|
||||
write_completion: WriteCompletion,
|
||||
) -> Result<()> {
|
||||
let opts = crate::object_api::ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion,
|
||||
..Default::default()
|
||||
};
|
||||
self.save_with_store_options(store, &opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_with_store_options(
|
||||
&mut self,
|
||||
store: Arc<ECStore>,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> Result<()> {
|
||||
self.parse_all_configs()?;
|
||||
self.validate_artifact_update(store.clone()).await?;
|
||||
let mut buf: Vec<u8> = vec![0; 4];
|
||||
|
||||
LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT);
|
||||
|
||||
LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION);
|
||||
LittleEndian::write_u16(
|
||||
&mut buf[2..4],
|
||||
if self.artifact_backup_retained {
|
||||
2
|
||||
} else {
|
||||
BUCKET_METADATA_VERSION
|
||||
},
|
||||
);
|
||||
|
||||
let data = self
|
||||
.marshal_msg()
|
||||
@@ -1141,21 +1209,52 @@ impl BucketMetadata {
|
||||
|
||||
buf.extend_from_slice(&data);
|
||||
|
||||
save_config_with_opts(
|
||||
store,
|
||||
self.save_file_path().as_str(),
|
||||
buf,
|
||||
&crate::object_api::ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
save_config_with_opts(store, self.save_file_path().as_str(), buf, opts).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_artifact_configuration(&self) -> Result<()> {
|
||||
if self.artifact_backup_retained && (self.artifact_base_prefix.is_none() || !self.bucket_creation_committed) {
|
||||
return Err(Error::other("artifact backup retention requires a committed managed bucket"));
|
||||
}
|
||||
if let Some(prefix) = &self.artifact_base_prefix {
|
||||
if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(prefix) || self.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other("invalid artifact bucket identity"));
|
||||
}
|
||||
let versioning = deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?;
|
||||
if !artifact_versioning_enabled(&versioning) {
|
||||
return Err(Error::other("artifact bucket versioning must remain enabled"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full metadata replacement must not change a generation's creation identity.
|
||||
pub(crate) async fn validate_artifact_update(&self, store: Arc<ECStore>) -> Result<()> {
|
||||
self.validate_artifact_configuration()?;
|
||||
// An existing empty/corrupt metadata object is not permission to create an identity.
|
||||
match read_config_preserve_empty(store, &self.save_file_path()).await {
|
||||
Ok(data) => {
|
||||
let previous = Self::decode_file(&data)?;
|
||||
previous.validate_artifact_configuration()?;
|
||||
if previous.artifact_backup_retained && !self.artifact_backup_retained {
|
||||
return Err(Error::other("artifact backup retention cannot be removed"));
|
||||
}
|
||||
// Deletion removes this record before recreation. A caller-supplied
|
||||
// incarnation cannot turn an existing bucket into a new generation.
|
||||
if previous.artifact_base_prefix != self.artifact_base_prefix
|
||||
|| (previous.artifact_base_prefix.is_some() && previous.bucket_incarnation_id != self.bucket_incarnation_id)
|
||||
{
|
||||
return Err(Error::other("artifact bucket identity is immutable"));
|
||||
}
|
||||
}
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_policy_config(&mut self) -> Result<()> {
|
||||
if !self.policy_config_json.is_empty() {
|
||||
self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?);
|
||||
@@ -1199,6 +1298,7 @@ impl BucketMetadata {
|
||||
/// `rustfs_bucket_metadata_parse_failed_total`, reflected in
|
||||
/// `rustfs_bucket_metadata_unparsable_current`, and logged at error level.
|
||||
pub(super) fn parse_all_configs(&mut self) -> Result<()> {
|
||||
self.validate_artifact_configuration()?;
|
||||
if let Err(e) = self.parse_policy_config() {
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1464,6 +1564,10 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
fn artifact_versioning_enabled(config: &VersioningConfiguration) -> bool {
|
||||
config.enabled() && !config.exclude_folders.unwrap_or(false) && config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_bucket_incarnation(api: Arc<ECStore>, bucket: &str) -> Result<Option<Uuid>> {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}");
|
||||
let data = match read_config_preserve_empty(api, &path).await {
|
||||
@@ -1560,9 +1664,7 @@ async fn read_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketM
|
||||
|
||||
let data = read_config(api, &file_path).await?;
|
||||
|
||||
BucketMetadata::check_header(&data)?;
|
||||
|
||||
let bm = BucketMetadata::unmarshal(&data[4..])?;
|
||||
let bm = BucketMetadata::decode_file(&data)?;
|
||||
|
||||
Ok(bm)
|
||||
}
|
||||
@@ -1586,6 +1688,98 @@ where
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn artifact_metadata_roundtrip_and_corrupt_prefix_rejection() {
|
||||
let mut metadata = BucketMetadata::new("artifacts");
|
||||
metadata.artifact_base_prefix = Some("images/".to_owned());
|
||||
let encoded = metadata.marshal_msg().expect("encode artifact metadata");
|
||||
let decoded = BucketMetadata::unmarshal(&encoded).expect("decode artifact metadata");
|
||||
assert_eq!(decoded.artifact_base_prefix, metadata.artifact_base_prefix, "ARTIFACT_METADATA_ROUNDTRIP");
|
||||
assert_eq!(decoded.bucket_incarnation_id, metadata.bucket_incarnation_id);
|
||||
let legacy = BucketMetadata::new("ordinary")
|
||||
.marshal_msg()
|
||||
.expect("encode ordinary metadata");
|
||||
assert!(
|
||||
BucketMetadata::unmarshal(&legacy)
|
||||
.expect("decode ordinary metadata")
|
||||
.artifact_base_prefix
|
||||
.is_none()
|
||||
);
|
||||
for prefix in ["", "../", "outputs", "a b/"] {
|
||||
let mut invalid = Vec::new();
|
||||
rmp::encode::write_map_len(&mut invalid, 1).expect("encode map");
|
||||
rmp::encode::write_str(&mut invalid, "ArtifactBasePrefix").expect("encode field");
|
||||
rmp::encode::write_str(&mut invalid, prefix).expect("encode invalid prefix");
|
||||
assert!(BucketMetadata::unmarshal(&invalid).is_err(), "ARTIFACT_CORRUPT_PREFIX");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_backup_retention_marker_rejects_false_and_duplicates() {
|
||||
for count in [1, 2] {
|
||||
let mut bytes = Vec::new();
|
||||
rmp::encode::write_map_len(&mut bytes, count).unwrap();
|
||||
for _ in 0..count {
|
||||
rmp::encode::write_str(&mut bytes, "ArtifactBackupRetained").unwrap();
|
||||
rmp::encode::write_bool(&mut bytes, count == 2).unwrap();
|
||||
}
|
||||
assert!(BucketMetadata::unmarshal(&bytes).is_err(), "O02P3_INVALID_RETENTION_MARKER");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_backup_retention_requires_matching_header_and_identity() {
|
||||
let mut metadata = BucketMetadata::new("retained");
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
metadata.bucket_creation_committed = true;
|
||||
metadata.artifact_base_prefix = Some("outputs/".to_owned());
|
||||
metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
|
||||
let file = |metadata: &BucketMetadata, version: u16| {
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(&BUCKET_METADATA_FORMAT.to_le_bytes());
|
||||
bytes.extend_from_slice(&version.to_le_bytes());
|
||||
bytes.extend_from_slice(&metadata.marshal_msg().unwrap());
|
||||
bytes
|
||||
};
|
||||
assert!(BucketMetadata::decode_file(&file(&metadata, 1)).is_ok(), "O02P3_ORDINARY_V1_COMPATIBLE");
|
||||
assert!(BucketMetadata::decode_file(&file(&metadata, 2)).is_err(), "O02P3_MARKER_REQUIRED");
|
||||
metadata.artifact_backup_retained = true;
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 1)).is_err(),
|
||||
"O02P3_PROTECTION_CANNOT_DOWNGRADE_HEADER"
|
||||
);
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 2))
|
||||
.unwrap()
|
||||
.artifact_backup_retained,
|
||||
"O02P3_PROTECTED_V2_ROUNDTRIP"
|
||||
);
|
||||
metadata.artifact_base_prefix = None;
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 2)).is_err(),
|
||||
"O02P3_PROTECTION_REQUIRES_MANAGED_BUCKET"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_versioning_cannot_be_disabled_or_excluded() {
|
||||
let mut metadata = BucketMetadata::new("artifacts");
|
||||
metadata.artifact_base_prefix = Some("images/".to_owned());
|
||||
for xml in [
|
||||
"",
|
||||
"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>",
|
||||
"<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>",
|
||||
"<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>images/</Prefix></ExcludedPrefixes></VersioningConfiguration>",
|
||||
] {
|
||||
assert!(
|
||||
metadata
|
||||
.update_config(BUCKET_VERSIONING_CONFIG, xml.as_bytes().to_vec())
|
||||
.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1734: `StorageError::clone` rebuilds I/O errors from
|
||||
/// their text. The unreadable-config refusal must stay typed across that
|
||||
/// clone, or a cloned error stops mapping to its retryable S3 response.
|
||||
|
||||
@@ -293,8 +293,10 @@ pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>
|
||||
pub async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let lock = sys.write().await;
|
||||
lock.set(bucket, Arc::new(bm)).await;
|
||||
Ok(())
|
||||
if bucket != bm.name {
|
||||
return Err(Error::other("bucket metadata identity mismatch"));
|
||||
}
|
||||
lock.set_validated(bm).await
|
||||
}
|
||||
|
||||
pub async fn reload_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
@@ -585,6 +587,15 @@ pub(crate) async fn get_cached_bucket_incarnation_id_in(
|
||||
sys.get_bucket_incarnation_id(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_incarnation_metadata_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
sys.get_bucket_incarnation_metadata(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn set_bucket_metadata_in(ctx: &crate::runtime::instance::InstanceContext, bm: BucketMetadata) -> Result<()> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let lock = sys.read().await;
|
||||
@@ -2132,7 +2143,16 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
pub(crate) async fn set_validated(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
if bm.artifact_base_prefix.is_some() {
|
||||
bm.parse_all_configs()?;
|
||||
}
|
||||
bm.validate_artifact_update(self.object_store()).await?;
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2147,6 +2167,11 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn set_for_test(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.set(bucket, bm).await;
|
||||
}
|
||||
|
||||
/// Remove a bucket's cached metadata from the in-memory map.
|
||||
///
|
||||
/// Returns `true` if an entry was present. Reserved meta buckets are ignored.
|
||||
@@ -2651,12 +2676,16 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
Ok(self.get_bucket_incarnation_metadata(bucket).await?.bucket_incarnation_id)
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let authority = self.get_metadata_authority(bucket).await?;
|
||||
if let BucketMetadataAuthority::Authoritative(metadata) = &authority
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& !metadata.bucket_incarnation_id.is_nil()
|
||||
{
|
||||
return Ok(metadata.bucket_incarnation_id);
|
||||
return Ok(Arc::clone(metadata));
|
||||
}
|
||||
|
||||
match authority {
|
||||
@@ -2665,7 +2694,7 @@ impl BucketMetadataSys {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
Ok(metadata.bucket_incarnation_id)
|
||||
Ok(metadata)
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) | BucketMetadataAuthority::Fabricated => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
@@ -2993,7 +3022,8 @@ impl BucketMetadataSys {
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
|
||||
if persisted && (metadata.lock_enabled || metadata.artifact_base_prefix.is_some()) && !metadata.bucket_creation_committed
|
||||
{
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if persisted {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||
|
||||
pub mod artifact;
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
pub mod config_parse_mode;
|
||||
|
||||
@@ -235,6 +235,8 @@ impl QuotaChecker {
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
/// Return quota configuration with optional scanner usage, independently
|
||||
/// of the usage requirements enforced by write admission.
|
||||
pub async fn get_quota_stats(&self, bucket: &str) -> Result<(BucketQuota, Option<u64>), QuotaError> {
|
||||
// If bucket doesn't exist, return ConfigNotFound error
|
||||
if !self.bucket_exists(bucket).await {
|
||||
@@ -244,18 +246,13 @@ impl QuotaChecker {
|
||||
}
|
||||
|
||||
let quota = self.get_quota_config(bucket).await?;
|
||||
let current_usage = if quota.uses_durable_reservations() {
|
||||
self.report_only_usage(bucket).await
|
||||
} else {
|
||||
Some(self.get_real_time_usage(bucket).await?)
|
||||
};
|
||||
let current_usage = self.report_only_usage(bucket).await;
|
||||
|
||||
Ok((quota, current_usage))
|
||||
}
|
||||
|
||||
/// Scanner usage for a bucket the reservation ledger admits is reporting
|
||||
/// input only, so unavailable memory usage is reported as "unknown" and counted
|
||||
/// instead of becoming an error the write path would fail on.
|
||||
/// Best-effort usage for statistics and durable admission reports.
|
||||
/// Missing scanner coverage is unknown, not an error or a zero-sized bucket.
|
||||
async fn report_only_usage(&self, bucket: &str) -> Option<u64> {
|
||||
// The plain lookup: a report must not wait for an early snapshot reload.
|
||||
let usage = get_bucket_usage_memory(bucket).await;
|
||||
@@ -267,7 +264,7 @@ impl QuotaChecker {
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA,
|
||||
state = "usage_unknown",
|
||||
bucket = %bucket,
|
||||
"durable quota usage report degraded"
|
||||
"quota usage report degraded"
|
||||
);
|
||||
}
|
||||
usage
|
||||
@@ -742,8 +739,12 @@ mod tests {
|
||||
1,
|
||||
"fail-closed admission takes the early-reload lookup"
|
||||
);
|
||||
assert!(matches!(checker.get_quota_stats(&legacy).await, Err(QuotaError::UsageUnavailable { .. })));
|
||||
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&legacy), 2);
|
||||
let (_, usage) = checker
|
||||
.get_quota_stats(&legacy)
|
||||
.await
|
||||
.expect("legacy quota statistics are reporting only");
|
||||
assert_eq!(usage, None);
|
||||
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&legacy), 1);
|
||||
assert_eq!(
|
||||
crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket),
|
||||
0,
|
||||
@@ -894,7 +895,7 @@ mod tests {
|
||||
.update_bucket_metadata_config(&bucket, QUOTA_CONFIG_FILE, legacy_quota_json(8192))
|
||||
.await
|
||||
.expect("newer quota save");
|
||||
sys.read().await.set(bucket.clone(), stale).await;
|
||||
sys.read().await.set_for_test(bucket.clone(), stale).await;
|
||||
assert_eq!(
|
||||
checker
|
||||
.migrate_reservation_protocol(&bucket)
|
||||
|
||||
@@ -245,3 +245,86 @@ async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
|
||||
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
|
||||
stop_reporting_fixture(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_statistics_remain_available_without_a_complete_scanner_snapshot() {
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
for configuration in ["unconfigured", "unlimited", "snapshot"] {
|
||||
for source in ["missing", "legacy", "incomplete", "observed", "complete", "empty"] {
|
||||
let (_dirs, store, _) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-statistics-{}", Uuid::new_v4().simple());
|
||||
match configuration {
|
||||
"unconfigured" => store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket without a quota configuration"),
|
||||
"unlimited" => {
|
||||
create_bucket_with_quota_json(
|
||||
&store,
|
||||
&bucket,
|
||||
serde_json::to_vec(&BucketQuota::new(None)).expect("unlimited quota encoding"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
"snapshot" => create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
// An unregistered checker keeps legacy migration out of the
|
||||
// assertion: reporting must work before a protocol upgrade.
|
||||
let metadata_sys = BucketMetadataSys::new(store.clone());
|
||||
let (metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
|
||||
metadata_sys.set_for_test(bucket.clone(), metadata).await;
|
||||
let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
|
||||
let (snapshot_source, size) = if source == "empty" { ("complete", 0) } else { (source, 1000) };
|
||||
install_reporting_snapshot(store.clone(), &bucket, snapshot_source, size).await;
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
|
||||
let statistics = checker.get_quota_stats(&bucket).await;
|
||||
let early_reloads = crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket);
|
||||
let missing_bucket = format!("missing-{}", Uuid::new_v4().simple());
|
||||
let missing_statistics = checker.get_quota_stats(&missing_bucket).await;
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
|
||||
.await
|
||||
.expect("fixture cache cleanup");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
|
||||
let (quota, usage) = statistics.unwrap_or_else(|error| {
|
||||
panic!("configuration={configuration}, source={source}: quota statistics must remain available: {error}")
|
||||
});
|
||||
assert_eq!(quota.quota, (configuration == "snapshot").then_some(4096));
|
||||
assert_eq!(
|
||||
usage,
|
||||
(snapshot_source == "complete").then_some(size),
|
||||
"configuration={configuration}, source={source}: only known usage is numeric"
|
||||
);
|
||||
assert_eq!(early_reloads, 0, "read-only statistics must not force an early snapshot reload");
|
||||
assert!(
|
||||
matches!(missing_statistics, Err(QuotaError::ConfigNotFound { bucket }) if bucket == missing_bucket),
|
||||
"reporting must still reject a missing bucket"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_statistics_preserve_configuration_errors() {
|
||||
let (_dirs, store, _) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-invalid-{}", Uuid::new_v4().simple());
|
||||
create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await;
|
||||
let metadata_sys = BucketMetadataSys::new(store.clone());
|
||||
let (mut metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
|
||||
Arc::make_mut(&mut metadata).quota_config_json = b"{not-json".to_vec();
|
||||
metadata_sys.set_for_test(bucket.clone(), metadata).await;
|
||||
let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
|
||||
|
||||
let statistics = checker.get_quota_stats(&bucket).await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
|
||||
assert!(
|
||||
matches!(statistics, Err(QuotaError::InvalidConfig { .. })),
|
||||
"unavailable reporting must not hide malformed quota configuration"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -631,6 +631,17 @@ fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> R
|
||||
Ok((version, epoch))
|
||||
}
|
||||
|
||||
fn decode_mrf_lifecycle_reader_capability(expected_member: &str, result: &[u8]) -> Result<(u8, Uuid)> {
|
||||
let (&maximum_format_version, identity) = result
|
||||
.split_first()
|
||||
.ok_or_else(|| Error::other("peer returned a truncated MRF lifecycle reader capability"))?;
|
||||
if maximum_format_version == 0 || expected_member.trim().is_empty() {
|
||||
return Err(Error::other("peer returned an invalid MRF lifecycle reader capability"));
|
||||
}
|
||||
let epoch = decode_remote_version_state_capability(expected_member, identity)?;
|
||||
Ok((maximum_format_version, epoch))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
pub events: Vec<u8>,
|
||||
@@ -1964,6 +1975,16 @@ impl PeerRestClient {
|
||||
Ok((self.topology_member.clone(), supported_version, epoch))
|
||||
}
|
||||
|
||||
/// Reports one peer's reader support without authorizing a lifecycle writer or rollback.
|
||||
pub async fn probe_mrf_lifecycle_reader(&self, topology_fingerprint: String) -> Result<(String, u8, Uuid)> {
|
||||
let probe = rustfs_protos::mrf_lifecycle_reader_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let (maximum_format_version, epoch) = decode_mrf_lifecycle_reader_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), maximum_format_version, epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_ilm_recovery_export(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::ilm_recovery_export_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
@@ -4333,6 +4354,49 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_lifecycle_reader_capability_decoder_preserves_the_advertised_version() {
|
||||
let epoch = Uuid::new_v4();
|
||||
for version in [1, 2, u8::MAX] {
|
||||
let result = rustfs_protos::encode_mrf_lifecycle_reader_capability(version, "node-a:9000", epoch.as_bytes())
|
||||
.expect("capability frame");
|
||||
assert_eq!(
|
||||
decode_mrf_lifecycle_reader_capability("node-a:9000", &result).expect("reported capability"),
|
||||
(version, epoch)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_lifecycle_reader_capability_decoder_rejects_unbound_or_malformed_identity() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let valid =
|
||||
rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "node-a:9000", epoch.as_bytes()).expect("capability frame");
|
||||
let mut invalid = vec![Vec::new(), vec![2], valid[..valid.len() - 1].to_vec()];
|
||||
let mut zero = valid.clone();
|
||||
zero[0] = 0;
|
||||
invalid.push(zero);
|
||||
let mut trailing = valid.clone();
|
||||
trailing.push(0);
|
||||
invalid.push(trailing);
|
||||
let mut length_overflow = valid.clone();
|
||||
length_overflow[1..9].copy_from_slice(&u64::MAX.to_be_bytes());
|
||||
invalid.push(length_overflow);
|
||||
let mut invalid_utf8 = valid.clone();
|
||||
invalid_utf8[9] = 0xff;
|
||||
invalid.push(invalid_utf8);
|
||||
invalid.push(
|
||||
rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "node-a:9000", Uuid::nil().as_bytes())
|
||||
.expect("nil epoch fixture"),
|
||||
);
|
||||
for result in invalid {
|
||||
assert!(decode_mrf_lifecycle_reader_capability("node-a:9000", &result).is_err());
|
||||
}
|
||||
assert!(decode_mrf_lifecycle_reader_capability("node-b:9000", &valid).is_err());
|
||||
let empty = rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "", epoch.as_bytes()).expect("empty member fixture");
|
||||
assert!(decode_mrf_lifecycle_reader_capability("", &empty).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_decoder_fails_closed() {
|
||||
let epoch = Uuid::new_v4();
|
||||
|
||||
@@ -2375,7 +2375,300 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
file_info.validate_for_metadata_read().map_err(Into::into)
|
||||
}
|
||||
|
||||
fn validate_artifact_repair_seal_response(
|
||||
request: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest,
|
||||
response: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse,
|
||||
) -> Result<()> {
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
|
||||
.map_err(|_| Error::other("artifact repair seal response is too large"))?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
.map_err(|_| Error::other("artifact repair seal response proof is invalid"))?;
|
||||
if !response.success {
|
||||
return Err(response.error.clone().unwrap_or_default().into());
|
||||
}
|
||||
let expected_disk = Uuid::from_slice(&request.expected_disk_id).ok().filter(|id| !id.is_nil());
|
||||
if response.error.is_some() || expected_disk.is_none() || Uuid::from_slice(&response.disk_id).ok() != expected_disk {
|
||||
return Err(Error::other("artifact repair seal response disk identity mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) async fn rename_artifact_repair_data(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
expected_incarnation: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"artifact_repair_rename_data",
|
||||
|| async {
|
||||
let body = ArtifactRepairRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: source.0.to_owned(),
|
||||
src_path: source.1.to_owned(),
|
||||
dst_volume: destination.0.to_owned(),
|
||||
dst_path: destination.1.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair rename request is too large"))?;
|
||||
let mut request = Request::new(body);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
// Unimplemented is terminal: never downgrade repair publication to PUT or legacy RPCs.
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.artifact_repair_rename_data(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn write_artifact_repair_metadata(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
target: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
expected_incarnation: Uuid,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"artifact_repair_write_metadata",
|
||||
|| async {
|
||||
let body = ArtifactRepairWriteMetadataRequest {
|
||||
metadata: Some(WriteMetadataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: target.0.to_owned(),
|
||||
path: target.1.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_write_metadata_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair metadata request is too large"))?;
|
||||
let mut request = Request::new(body);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.artifact_repair_write_metadata(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// One authenticated scan (or explicitly distinct read-only identity probe).
|
||||
/// Partial observations are never retried after output has begun.
|
||||
pub async fn inventory_artifact_attempt(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
target: (&str, Uuid),
|
||||
prefix: &str,
|
||||
phase: rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase,
|
||||
deadline: tokio::time::Instant,
|
||||
output: tokio::sync::mpsc::Sender<crate::disk::ArtifactInventoryFrame>,
|
||||
) -> Result<()> {
|
||||
use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
use rustfs_protos::{
|
||||
artifact_inventory as protocol,
|
||||
proto_gen::node_service::{ArtifactInventoryRequest, artifact_inventory_frame::Payload},
|
||||
};
|
||||
let (bucket, incarnation) = target;
|
||||
let duration = deadline
|
||||
.checked_duration_since(tokio::time::Instant::now())
|
||||
.ok_or(Error::Timeout)?;
|
||||
let body = ArtifactInventoryRequest {
|
||||
protocol_version: protocol::VERSION,
|
||||
invocation_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
disk: self.endpoint.to_string(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: prefix.to_owned(),
|
||||
timeout_millis: u64::try_from(duration.as_millis().min(u128::from(protocol::MAX_TIMEOUT_MILLIS)))
|
||||
.map_err(Error::other)?,
|
||||
phase: phase.into(),
|
||||
};
|
||||
protocol::validate_request(&body).map_err(Error::other)?;
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
let canonical = protocol::canonical_request(&body).map_err(Error::other)?;
|
||||
let mut request = Request::new(body.clone());
|
||||
request.set_timeout(duration);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let mut stream = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(protocol::MAX_FRAME_BYTES)
|
||||
.inventory_artifact_attempt(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
let mut state = protocol::StreamState::new();
|
||||
let mut terminal = None;
|
||||
while let Some(frame) = stream.message().await? {
|
||||
state
|
||||
.accept(&body, &frame, |canonical, proof| {
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(canonical, proof).is_ok()
|
||||
})
|
||||
.map_err(Error::other)?;
|
||||
match frame.payload {
|
||||
Some(Payload::Entries(entries)) => {
|
||||
let versions = entries
|
||||
.versions
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
Ok(ArtifactInventoryVersion {
|
||||
key: entry.key,
|
||||
version_id: Uuid::from_slice(&entry.version_id).map_err(Error::other)?,
|
||||
delete_marker: entry.delete_marker,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
output
|
||||
.send(ArtifactInventoryFrame::Entries {
|
||||
sequence: frame.sequence,
|
||||
versions,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
}
|
||||
Some(Payload::Complete(complete)) => terminal = Some(complete.entries),
|
||||
Some(Payload::Verified(_)) => {}
|
||||
None => return Err(Error::other("artifact inventory payload is absent")),
|
||||
}
|
||||
}
|
||||
// A valid Complete followed by a transport failure or extra frame is
|
||||
// still incomplete. Publish the local terminal only after clean EOF.
|
||||
state.finish().map_err(Error::other)?;
|
||||
if let Some(entries) = terminal {
|
||||
output
|
||||
.send(ArtifactInventoryFrame::Complete {
|
||||
disk_id: expected_disk,
|
||||
incarnation,
|
||||
attempt_prefix: prefix.to_owned(),
|
||||
entries,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::Timeout)?
|
||||
}
|
||||
|
||||
pub(crate) async fn seal_artifact_puts(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"seal_artifact_puts",
|
||||
|| async {
|
||||
let body = SealArtifactAttemptPutsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: attempt_prefix.to_owned(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact seal request is too large"))?;
|
||||
let mut request = Request::new(body.clone());
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.seal_artifact_attempt_puts(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&body, &response)
|
||||
.map_err(|_| Error::other("artifact seal response is too large"))?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
.map_err(|_| Error::other("artifact seal response proof is invalid"))?;
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.error.is_some() || Uuid::from_slice(&response.disk_id).ok() != Some(expected_disk) {
|
||||
return Err(Error::other("artifact seal response disk identity mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn seal_artifact_repairs(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
|
||||
if expected_disk.is_nil() || incarnation.is_nil() {
|
||||
return Err(Error::other("artifact repair seal requires non-nil disk and incarnation identities"));
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"seal_artifact_repairs",
|
||||
|| async {
|
||||
let body = SealArtifactAttemptPutsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: attempt_prefix.to_owned(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair seal request is too large"))?;
|
||||
let mut request = Request::new(body.clone());
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.seal_artifact_attempt_repairs(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
validate_artifact_repair_seal_response(&body, &response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -2385,20 +2678,39 @@ impl RemoteDisk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
self.rename_data_borrowed_with_fence((src_volume, src_path), fi, (dst_volume, dst_path), None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
target: (&str, &str),
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
artifact_incarnation: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
let (src_volume, src_path) = source;
|
||||
let (dst_volume, dst_path) = target;
|
||||
if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path)
|
||||
&& let Some(identity) = scope.receiver_identity()
|
||||
{
|
||||
scope.check().map_err(Error::other)?;
|
||||
if artifact_incarnation.is_some() || scanner_publication_lease_token.is_some() {
|
||||
return Err(Error::other("artifact repair cannot use a publication lease"));
|
||||
}
|
||||
let expected = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
|
||||
let result = self
|
||||
.rename_artifact_repair_data(expected, (src_volume, src_path), fi, (dst_volume, dst_path), identity.incarnation)
|
||||
.await;
|
||||
scope.check().map_err(Error::other)?;
|
||||
return result;
|
||||
}
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2421,8 +2733,11 @@ impl RemoteDisk {
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
|
||||
.map(|scope| scope.incarnation.as_bytes().to_vec().into())
|
||||
bucket_incarnation_id: artifact_incarnation
|
||||
.or_else(|| {
|
||||
crate::store::bucket_heal_scope_for_object(dst_volume, dst_path).map(|scope| scope.incarnation)
|
||||
})
|
||||
.map(|incarnation| incarnation.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
@@ -2435,7 +2750,11 @@ impl RemoteDisk {
|
||||
.map(|token| token.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
let canonical_body = if artifact_incarnation.is_some() {
|
||||
rustfs_protos::canonical_artifact_rename_data_request_body(request.get_ref())
|
||||
} else {
|
||||
rustfs_protos::canonical_rename_data_request_body(request.get_ref())
|
||||
};
|
||||
let incarnation_bound = !request.get_ref().bucket_incarnation_id.is_empty();
|
||||
let quota_fenced = rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
@@ -2458,7 +2777,9 @@ impl RemoteDisk {
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if quota_fenced {
|
||||
if artifact_incarnation.is_some() {
|
||||
client.artifact_rename_data(request).await
|
||||
} else if quota_fenced {
|
||||
// The method itself proves quota-token enforcement; an old peer must fail closed.
|
||||
client.rename_data_with_quota_fence(request).await
|
||||
} else if incarnation_bound {
|
||||
@@ -3204,6 +3525,21 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(volume, path)
|
||||
&& let Some(identity) = scope.receiver_identity()
|
||||
{
|
||||
scope.check().map_err(Error::other)?;
|
||||
let expected = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
|
||||
let result = self
|
||||
.write_artifact_repair_metadata(expected, (volume, path), &fi, identity.incarnation)
|
||||
.await;
|
||||
scope.check().map_err(Error::other)?;
|
||||
return result;
|
||||
}
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -4553,6 +4889,83 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_old_peer_unimplemented_never_downgrades_mutations() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("loopback test peer");
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse(&format!("{}/data/rustfs0", peer.addr)).expect("test disk endpoint"),
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: 0,
|
||||
};
|
||||
let disk = RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("remote test disk");
|
||||
let expected_disk = Uuid::new_v4();
|
||||
let incarnation = Uuid::new_v4();
|
||||
let info = FileInfo::default();
|
||||
let rename = disk
|
||||
.rename_artifact_repair_data(
|
||||
expected_disk,
|
||||
("tmp", "staged"),
|
||||
&info,
|
||||
("artifacts", "outputs/tenant/run/attempt/image"),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot publish repair data");
|
||||
let metadata = disk
|
||||
.write_artifact_repair_metadata(expected_disk, ("artifacts", "outputs/tenant/run/attempt/image"), &info, incarnation)
|
||||
.await
|
||||
.expect_err("old peer cannot publish repair metadata");
|
||||
let seal = disk
|
||||
.seal_artifact_repairs(expected_disk, "artifacts", incarnation, "outputs/tenant/run/attempt")
|
||||
.await
|
||||
.expect_err("old peer cannot confirm repair closure");
|
||||
for error in [rename, metadata, seal] {
|
||||
let DiskError::Io(error) = error else {
|
||||
panic!("expected gRPC status error");
|
||||
};
|
||||
let status = error
|
||||
.get_ref()
|
||||
.and_then(|error| error.downcast_ref::<crate::disk::error::RpcStatusError>())
|
||||
.expect("preserved gRPC status");
|
||||
assert_eq!(status.status().code(), Code::Unimplemented);
|
||||
}
|
||||
let paths = peer.peer.rpc_paths.lock().expect("RPC inventory").clone();
|
||||
for method in [
|
||||
"ArtifactRepairRenameData",
|
||||
"ArtifactRepairWriteMetadata",
|
||||
"SealArtifactAttemptRepairs",
|
||||
] {
|
||||
assert_eq!(
|
||||
paths.iter().filter(|path| path.ends_with(&format!("/{method}"))).count(),
|
||||
1,
|
||||
"each repair method is sent once"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
paths.iter().all(|path| matches!(
|
||||
path.rsplit('/').next(),
|
||||
Some("ArtifactRepairRenameData" | "ArtifactRepairWriteMetadata" | "SealArtifactAttemptRepairs" | "Ping")
|
||||
)),
|
||||
"repair cannot downgrade to a legacy mutation: {paths:?}"
|
||||
);
|
||||
disk.cancel_token.cancel();
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -4648,6 +5061,93 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_repair_seal_response_requires_proof_full_request_and_physical_identity() {
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
Error as WireError, SealArtifactAttemptPutsRequest, SealArtifactAttemptPutsResponse,
|
||||
};
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let request = SealArtifactAttemptPutsRequest {
|
||||
disk: "disk-a".into(),
|
||||
bucket: "artifacts".into(),
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
attempt_prefix: "outputs/tenant/run/attempt".into(),
|
||||
expected_disk_id: vec![2; 16].into(),
|
||||
};
|
||||
let mut response = SealArtifactAttemptPutsResponse {
|
||||
success: true,
|
||||
disk_id: request.expected_disk_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let sign = |request: &SealArtifactAttemptPutsRequest, response: &mut SealArtifactAttemptPutsResponse| {
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
|
||||
.expect("repair response canonical");
|
||||
response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&canonical)
|
||||
.expect("repair response proof")
|
||||
.into();
|
||||
};
|
||||
assert!(validate_artifact_repair_seal_response(&request, &response).is_err(), "proof is mandatory");
|
||||
sign(&request, &mut response);
|
||||
validate_artifact_repair_seal_response(&request, &response).expect("signed exact physical receiver");
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.disk = "other-disk".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket = "other-bucket".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = vec![3; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.attempt_prefix = "other-attempt".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = vec![4; 16].into(),
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&changed, &response).is_err(),
|
||||
"another request cannot reuse the proof"
|
||||
);
|
||||
}
|
||||
let legacy = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&request, &response).expect("PUT domain");
|
||||
let mut legacy_response = response.clone();
|
||||
legacy_response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&legacy)
|
||||
.expect("PUT proof")
|
||||
.into();
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &legacy_response).is_err(),
|
||||
"PUT proof cannot acknowledge repair seal"
|
||||
);
|
||||
for disk in [vec![], vec![0; 16], vec![3; 16], vec![1; 15]] {
|
||||
let mut changed = response.clone();
|
||||
changed.disk_id = disk.into();
|
||||
sign(&request, &mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"signed wrong or absent physical identity is invalid"
|
||||
);
|
||||
}
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.success = false,
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.error = Some(WireError::default()),
|
||||
] {
|
||||
let mut changed = response.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"response fields are bound"
|
||||
);
|
||||
sign(&request, &mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"signed failure or inconsistent success is invalid"
|
||||
);
|
||||
}
|
||||
let mut nil_request = request;
|
||||
nil_request.expected_disk_id = vec![0; 16].into();
|
||||
let mut nil_response = response;
|
||||
nil_response.disk_id = nil_request.expected_disk_id.clone();
|
||||
sign(&nil_request, &mut nil_response);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&nil_request, &nil_response).is_err(),
|
||||
"nil physical identity is never a receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() {
|
||||
let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2)
|
||||
@@ -5038,6 +5538,7 @@ mod tests {
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
rpc_paths: Arc<StdMutex<Vec<String>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
@@ -5057,6 +5558,7 @@ mod tests {
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
rpc_paths: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -5112,6 +5614,11 @@ mod tests {
|
||||
}
|
||||
|
||||
fn call(&mut self, request: http::Request<B>) -> Self::Future {
|
||||
self.peer
|
||||
.rpc_paths
|
||||
.lock()
|
||||
.expect("RPC path inventory")
|
||||
.push(request.uri().path().to_owned());
|
||||
match request.uri().path() {
|
||||
"/node_service.NodeService/DiskInfo" => {
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -438,6 +438,64 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: false,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation_no_lock<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if !guard.matches(RUSTFS_META_BUCKET, file) {
|
||||
return Err(Error::InvalidArgument(
|
||||
RUSTFS_META_BUCKET.to_owned(),
|
||||
file.to_owned(),
|
||||
"locked observation requires a matching write owner".to_owned(),
|
||||
));
|
||||
}
|
||||
let lock_lost_error = || Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
bucket: RUSTFS_META_BUCKET.to_owned(),
|
||||
object: file.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: true,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
let result = crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -484,10 +542,22 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
|
||||
let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Preserve an empty config payload, its metadata, and caller-supplied read options.
|
||||
pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, None).await
|
||||
}
|
||||
|
||||
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -516,15 +586,13 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(
|
||||
read_config_preserve_empty_with_metadata_opts(
|
||||
api,
|
||||
file,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2919,9 +2987,9 @@ mod tests {
|
||||
config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
|
||||
heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
|
||||
new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
|
||||
read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
|
||||
save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_transaction_lock_path, storage_class_kvs_mut,
|
||||
read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
|
||||
read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, heal, notify, oidc, scanner};
|
||||
use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||
@@ -5396,6 +5464,7 @@ mod tests {
|
||||
heal_replacement: Option<Vec<u8>>,
|
||||
heal_calls: AtomicUsize,
|
||||
write_calls: AtomicUsize,
|
||||
last_get_no_lock: AtomicBool,
|
||||
last_put_no_lock: AtomicBool,
|
||||
last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
|
||||
revision: AtomicUsize,
|
||||
@@ -5411,6 +5480,7 @@ mod tests {
|
||||
heal_replacement,
|
||||
heal_calls: AtomicUsize::new(0),
|
||||
write_calls: AtomicUsize::new(0),
|
||||
last_get_no_lock: AtomicBool::new(false),
|
||||
last_put_no_lock: AtomicBool::new(false),
|
||||
last_put_preconditions: Mutex::new(None),
|
||||
revision: AtomicUsize::new(1),
|
||||
@@ -5466,8 +5536,9 @@ mod tests {
|
||||
_object: &str,
|
||||
_range: Option<HTTPRangeSpec>,
|
||||
_h: HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
|
||||
let data = match &*self.state.lock().expect("state lock poisoned") {
|
||||
RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
|
||||
RecoveryReadState::Blob(data) => data.clone(),
|
||||
@@ -5779,6 +5850,325 @@ mod tests {
|
||||
assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
type ObservationEofGate = (Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
struct ObservationStream {
|
||||
inner: Cursor<Vec<u8>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
fail: bool,
|
||||
eof_gate: Option<ObservationEofGate>,
|
||||
}
|
||||
|
||||
impl AsyncRead for ObservationStream {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.fail {
|
||||
return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
|
||||
}
|
||||
if self.inner.position() == u64::try_from(self.inner.get_ref().len()).expect("fixture length")
|
||||
&& let Some((ready, release)) = self.eof_gate.as_mut()
|
||||
{
|
||||
if let Some(ready) = ready.take() {
|
||||
ready.send(()).expect("EOF observer remains active");
|
||||
}
|
||||
match std::future::Future::poll(Pin::new(release), cx) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(result) => result.expect("EOF gate released"),
|
||||
}
|
||||
self.eof_gate = None;
|
||||
}
|
||||
let before = buf.filled().len();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationStore {
|
||||
body: Vec<u8>,
|
||||
declared_size: i64,
|
||||
error: Option<Error>,
|
||||
stream_error: bool,
|
||||
gets: AtomicUsize,
|
||||
eof_gate: std::sync::Mutex<Option<ObservationEofGate>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
options: std::sync::Mutex<Option<(bool, bool, bool)>>,
|
||||
writes: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Debug for ObservationStore {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ObservationStore {
|
||||
fn new(body: Vec<u8>, declared_size: i64) -> Self {
|
||||
Self {
|
||||
body,
|
||||
declared_size,
|
||||
error: None,
|
||||
stream_error: false,
|
||||
gets: AtomicUsize::new(0),
|
||||
eof_gate: std::sync::Mutex::new(None),
|
||||
consumed: Arc::new(AtomicUsize::new(0)),
|
||||
options: std::sync::Mutex::new(None),
|
||||
writes: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
_headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
assert_eq!(bucket, RUSTFS_META_BUCKET);
|
||||
assert_eq!(object, "config/observation.bin");
|
||||
self.gets.fetch_add(1, Ordering::SeqCst);
|
||||
assert!(range.is_none());
|
||||
*self.options.lock().expect("observation options") =
|
||||
Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
|
||||
if let Some(error) = &self.error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
Ok(GetObjectReader {
|
||||
stream: Box::new(ObservationStream {
|
||||
inner: Cursor::new(self.body.clone()),
|
||||
consumed: self.consumed.clone(),
|
||||
fail: self.stream_error,
|
||||
eof_gate: self.eof_gate.lock().expect("EOF gate").take(),
|
||||
}),
|
||||
object_info: ObjectInfo {
|
||||
size: self.declared_size,
|
||||
etag: Some("observation-etag".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.writes.fetch_add(1, Ordering::SeqCst);
|
||||
Err(Error::other("observation must not write"))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_preserves_options_and_typed_errors() {
|
||||
let object = "config/observation.bin";
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-options".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-options".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert!(result.is_ok(), "I06L_READ_FLAGS: {result:?}");
|
||||
assert!(*store.options.lock().expect("options") == Some((true, true, false)), "I06L_READ_FLAGS");
|
||||
assert_eq!(result.expect("raw observation").0, b"raw");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
for (bucket, key) in [(RUSTFS_META_BUCKET, "config/other.bin"), ("wrong-bucket", object)] {
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-target".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(bucket, key),
|
||||
"locked-observation-target".to_owned(),
|
||||
);
|
||||
let wrong = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("wrong-target real owner");
|
||||
let store = Arc::new(ObservationStore::new(Vec::new(), 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &wrong).await;
|
||||
assert!(matches!(result, Err(Error::InvalidArgument(..))), "I06L_TARGET_REJECTED");
|
||||
assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_TARGET_REJECTED");
|
||||
}
|
||||
for error in [
|
||||
Error::FileNotFound,
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_owned(), object.to_owned()),
|
||||
Error::ErasureReadQuorum,
|
||||
] {
|
||||
let expected = if error == Error::FileNotFound || matches!(error, Error::ObjectNotFound(..)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
error.clone()
|
||||
};
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(error);
|
||||
let store = Arc::new(fixture);
|
||||
assert_eq!(
|
||||
super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("typed error"),
|
||||
expected
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let err = super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("stream failure");
|
||||
assert!(err.to_string().contains("observation stream failed"), "I06L_TYPED_STREAM_ERROR");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &guard).await;
|
||||
assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_STREAM_BOUND");
|
||||
assert!(store.consumed.load(Ordering::SeqCst) == 9, "I06L_STREAM_BOUND");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_rejects_actual_owner_loss_before_and_after_read() {
|
||||
let object = "config/observation.bin";
|
||||
for after_read in [false, true] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let client = Arc::new(RefreshControlledClient::new(manager));
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::new("locked-observation-loss".to_owned(), client.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-loss".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("real distributed owner");
|
||||
let signal = guard.lock_lost_signal().expect("acquired distributed owner signal");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
if after_read {
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
*store.eof_gate.lock().expect("EOF gate") = Some((Some(ready_tx), release_rx));
|
||||
let mut read = Box::pin(super::read_config_observation_no_lock(store.clone(), object, 3, &guard));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
tokio::select! {
|
||||
ready = ready_rx => ready.expect("read reached EOF"),
|
||||
result = &mut read => assert_eq!(false, true, "I06L_LOSS_AFTER_READ: returned before EOF gate: {result:?}"),
|
||||
}
|
||||
}).await.expect("EOF watchdog");
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert!(guard.is_lock_lost(), "I06L_LOSS_AFTER_READ");
|
||||
release_tx.send(()).expect("release EOF");
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(60), read)
|
||||
.await
|
||||
.expect("read watchdog");
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
"I06L_LOSS_AFTER_READ"
|
||||
);
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst), 1);
|
||||
} else {
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert!(guard.is_lock_lost(), "I06L_LOSS_BEFORE_GET");
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
"I06L_LOSS_BEFORE_GET"
|
||||
);
|
||||
assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_LOSS_BEFORE_GET");
|
||||
}
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_preserves_options_and_errors() {
|
||||
for body in [Vec::new(), b"raw config".to_vec()] {
|
||||
let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
|
||||
let (observed, info) = result.expect("successful observation");
|
||||
assert_eq!(observed, body);
|
||||
assert_eq!(info.etag.as_deref(), Some("observation-etag"));
|
||||
assert_eq!(
|
||||
*store.options.lock().expect("options"),
|
||||
Some((true, false, false)),
|
||||
"I05_REPAIR_SUPPRESSED"
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
for (source, expected) in [
|
||||
(Error::FileNotFound, Error::ConfigNotFound),
|
||||
(
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
|
||||
Error::ConfigNotFound,
|
||||
),
|
||||
(Error::ErasureReadQuorum, Error::ErasureReadQuorum),
|
||||
] {
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(source);
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert_eq!(result.expect_err("read error must propagate"), expected);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
let error = result.expect_err("stream error must propagate");
|
||||
assert!(error.to_string().contains("observation stream failed"));
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_enforces_byte_limit() {
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert!(result.is_err(), "I05_BOUND_ENFORCED");
|
||||
assert_eq!(
|
||||
result.expect_err("bounded observation error"),
|
||||
Error::CorruptedFormat,
|
||||
"I05_BOUND_ENFORCED"
|
||||
);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
|
||||
assert_eq!(result.expect("exact limit").0, vec![7; 8]);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_distinguishes_empty_object() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
@@ -5799,6 +6189,60 @@ mod tests {
|
||||
assert!(data.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
for no_lock in [false, true] {
|
||||
let opts = ObjectOptions {
|
||||
no_lock,
|
||||
..Default::default()
|
||||
};
|
||||
let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
|
||||
.await
|
||||
.expect("an existing empty config must retain its payload and metadata");
|
||||
assert!(data.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
assert_eq!(info.actual_size, 0);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
|
||||
}
|
||||
let err = read_config(store, "config/empty.json")
|
||||
.await
|
||||
.expect_err("the existing reader must still treat empty configs as missing");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
|
||||
let payload = b"original config bytes".to_vec();
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
|
||||
let (data, info) =
|
||||
read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("a nonempty config must retain its payload and metadata");
|
||||
assert_eq!(data, payload);
|
||||
let size = i64::try_from(payload.len()).expect("test payload size fits i64");
|
||||
assert_eq!(info.size, size);
|
||||
assert_eq!(info.actual_size, size);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_preserves_read_errors() {
|
||||
let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("physical absence must remain ConfigNotFound");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a quorum failure must retain its typed read error");
|
||||
assert!(matches!(err, Error::ErasureReadQuorum));
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl StorageAdminApi for RecoveryMockStore {
|
||||
type BackendInfo = rustfs_madmin::BackendInfo;
|
||||
|
||||
@@ -17076,6 +17076,7 @@ impl ECStore {
|
||||
&[],
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: None,
|
||||
lifecycle_owner: None,
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
|
||||
object_mutation_fence: Some(&object_fence),
|
||||
@@ -17344,7 +17345,7 @@ impl ECStore {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
|
||||
let bucket_incarnation_fence = match expected_bucket_incarnation_id {
|
||||
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||
Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
|
||||
None => None,
|
||||
};
|
||||
|
||||
@@ -18280,9 +18281,8 @@ impl ECStore {
|
||||
&cleanup_transferred_tier_versions,
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: expected_bucket_incarnation_id,
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
lifecycle_owner: bucket_incarnation_fence.as_ref(),
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: None,
|
||||
object_mutation_fence: Some(&source_cleanup_mutation_fence),
|
||||
},
|
||||
@@ -29408,6 +29408,7 @@ mod pools_tests {
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -499,6 +499,7 @@ mod decommission_lock_order_tests {
|
||||
),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
(store, refresh_calls)
|
||||
|
||||
@@ -99,6 +99,33 @@ pub struct Sets {
|
||||
ctx: Arc<InstanceContext>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Sets {
|
||||
pub(crate) fn from_test_set(
|
||||
set: Arc<SetDisks>,
|
||||
endpoints: PoolEndpoints,
|
||||
format: FormatV3,
|
||||
pool_idx: usize,
|
||||
parity_count: usize,
|
||||
ctx: Arc<InstanceContext>,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
id: format.id,
|
||||
disk_set: vec![set],
|
||||
pool_idx,
|
||||
set_count: 1,
|
||||
set_drive_count: endpoints.drives_per_set,
|
||||
endpoints,
|
||||
format,
|
||||
parity_count,
|
||||
default_parity_count: parity_count,
|
||||
distribution_algo: DistributionAlgoVersion::V1,
|
||||
exit_signal: None,
|
||||
ctx,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Sets {
|
||||
fn drop(&mut self) {
|
||||
if let Some(exit_signal) = self.exit_signal.take() {
|
||||
@@ -800,6 +827,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
bucket_incarnation_fence: dst_opts.bucket_incarnation_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ use crate::storage_api_contracts::{
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
|
||||
use crate::store::{BucketIncarnationFenceGuard, DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
@@ -1278,6 +1278,7 @@ pub(crate) enum SourceCleanupError {
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct SourceCleanupBucketFence<'a> {
|
||||
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
|
||||
pub(crate) lifecycle_owner: Option<&'a Arc<BucketIncarnationFenceGuard>>,
|
||||
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
pub(crate) namespace_lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
|
||||
@@ -1486,9 +1487,10 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
)
|
||||
};
|
||||
|
||||
if bucket_fence
|
||||
.lifecycle_guard
|
||||
.is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
|
||||
if bucket_fence.lifecycle_owner.is_some_and(|owner| owner.is_lock_lost())
|
||||
|| bucket_fence
|
||||
.lifecycle_guard
|
||||
.is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
|
||||
{
|
||||
return Err(SourceCleanupError::Storage(Error::other(format!(
|
||||
"{op_label}: bucket incarnation fence was lost before source cleanup"
|
||||
@@ -1527,6 +1529,9 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
|
||||
object_mutation_fence.add_namespace_lock_fence(&mut opts);
|
||||
}
|
||||
if let Some(owner) = bucket_fence.lifecycle_owner {
|
||||
owner.attach_owner(&mut opts);
|
||||
}
|
||||
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
|
||||
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
|
||||
}
|
||||
|
||||
@@ -216,7 +216,7 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
|
||||
// Stale or empty cache: refresh through the TTL-bounded loader. A failed
|
||||
// refresh carries the previous baseline forward, so quota admission keeps
|
||||
// its last grounded values through a backend read outage.
|
||||
let _ = load_data_usage_from_backend_cached(store).await;
|
||||
let _ = project_data_usage_from_backend_cached(store, |_| ()).await;
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
cache
|
||||
.as_ref()
|
||||
@@ -237,31 +237,34 @@ struct CachedDataUsageSnapshot {
|
||||
}
|
||||
|
||||
impl CachedDataUsageSnapshot {
|
||||
fn result(&self) -> Result<DataUsageInfo, Error> {
|
||||
fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
|
||||
self.info
|
||||
.clone()
|
||||
.as_ref()
|
||||
.map(project)
|
||||
.ok_or_else(|| Error::other("data usage snapshot load recently failed"))
|
||||
}
|
||||
}
|
||||
|
||||
fn fresh_cached_data_usage_snapshot(
|
||||
fn fresh_cached_data_usage_snapshot<T>(
|
||||
cache: &Option<CachedDataUsageSnapshot>,
|
||||
now: tokio::time::Instant,
|
||||
ttl: Duration,
|
||||
) -> Option<Result<DataUsageInfo, Error>> {
|
||||
project: impl FnOnce(&DataUsageInfo) -> T,
|
||||
) -> Option<Result<T, Error>> {
|
||||
cache
|
||||
.as_ref()
|
||||
.filter(|cached| now.duration_since(cached.loaded_at) < ttl)
|
||||
.map(CachedDataUsageSnapshot::result)
|
||||
.map(|cached| cached.result(project))
|
||||
}
|
||||
|
||||
fn cache_data_usage_snapshot_result(
|
||||
fn cache_data_usage_snapshot_result<T>(
|
||||
cache: &mut Option<CachedDataUsageSnapshot>,
|
||||
result: Result<LoadedUsageBaseline, Error>,
|
||||
loaded_at: tokio::time::Instant,
|
||||
refresh_generation: u64,
|
||||
current_generation: u64,
|
||||
) -> Option<Result<DataUsageInfo, Error>> {
|
||||
project: impl FnOnce(&DataUsageInfo) -> T,
|
||||
) -> Option<Result<T, Error>> {
|
||||
if current_generation != refresh_generation {
|
||||
return None;
|
||||
}
|
||||
@@ -280,12 +283,13 @@ fn cache_data_usage_snapshot_result(
|
||||
degraded_baseline.entry(bucket.clone()).or_insert(*size);
|
||||
}
|
||||
}
|
||||
let projected = project(&info);
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info.clone()),
|
||||
info: Some(info),
|
||||
loaded_at,
|
||||
degraded_baseline,
|
||||
});
|
||||
Ok(info)
|
||||
Ok(projected)
|
||||
}
|
||||
Err(e) => {
|
||||
// Keep the previous baseline through a failed refresh: quota
|
||||
@@ -1486,12 +1490,21 @@ async fn normalize_loaded_data_usage(
|
||||
/// `save_data_usage_in_backend` invalidates the cache so a fresh scanner save
|
||||
/// is visible immediately in the saving process.
|
||||
pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
project_data_usage_from_backend_cached(store, DataUsageInfo::clone).await
|
||||
}
|
||||
|
||||
// Project under the cache guard so scalar readers never copy the snapshot's
|
||||
// bucket maps. The owned API shares the same TTL, refresh and invalidation path.
|
||||
async fn project_data_usage_from_backend_cached<T>(
|
||||
store: Arc<ECStore>,
|
||||
project: impl Fn(&DataUsageInfo) -> T,
|
||||
) -> Result<T, Error> {
|
||||
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
|
||||
loop {
|
||||
{
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, &project) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1503,7 +1516,7 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
|
||||
.await;
|
||||
{
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, &project) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1512,9 +1525,14 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
|
||||
let result = load_data_usage_from_backend_with_baseline(store.clone()).await;
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut cache = data_usage_snapshot_cache().write().await;
|
||||
if let Some(result) =
|
||||
cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation, data_usage_snapshot_generation())
|
||||
{
|
||||
if let Some(result) = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
result,
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
&project,
|
||||
) {
|
||||
return result;
|
||||
}
|
||||
drop(cache);
|
||||
@@ -1532,7 +1550,8 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
loop {
|
||||
{
|
||||
let cache = admin_data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, DataUsageInfo::clone)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1543,7 +1562,8 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
.await;
|
||||
{
|
||||
let cache = admin_data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, DataUsageInfo::clone)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1564,6 +1584,7 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
admin_data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
) {
|
||||
return result;
|
||||
}
|
||||
@@ -2277,8 +2298,10 @@ async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
|
||||
}
|
||||
|
||||
async fn persisted_bucket_usage(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
|
||||
let info = load_data_usage_from_backend_cached(store).await.ok()?;
|
||||
info.buckets_usage.get(bucket).map(|usage| usage.size)
|
||||
project_data_usage_from_backend_cached(store, |info| info.buckets_usage.get(bucket).map(|usage| usage.size))
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
async fn reload_persisted_bucket_usage(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
|
||||
@@ -2375,6 +2398,7 @@ where
|
||||
tokio::time::Instant::now(),
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
|_| (),
|
||||
)
|
||||
.is_some()
|
||||
}
|
||||
@@ -2891,6 +2915,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod snapshot_projection;
|
||||
|
||||
use super::*;
|
||||
use crate::layout::endpoints::EndpointServerPools;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
@@ -3334,6 +3360,7 @@ mod tests {
|
||||
pool_meta_save_gate: TokioMutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4572,15 +4599,29 @@ mod tests {
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache");
|
||||
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
|
||||
|
||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||
.expect("the failed load must remain cached within the TTL");
|
||||
let cached = fresh_cached_data_usage_snapshot(
|
||||
&cache,
|
||||
loaded_at + Duration::from_secs(1),
|
||||
Duration::from_secs(30),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("the failed load must remain cached within the TTL");
|
||||
assert!(cached.is_err());
|
||||
|
||||
assert!(fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(30), Duration::from_secs(30)).is_none());
|
||||
assert!(
|
||||
fresh_cached_data_usage_snapshot(
|
||||
&cache,
|
||||
loaded_at + Duration::from_secs(30),
|
||||
Duration::from_secs(30),
|
||||
DataUsageInfo::clone
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4601,14 +4642,20 @@ mod tests {
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
assert_snapshot_bucket(&first, "bucket");
|
||||
|
||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||
.expect("successful load must remain cached within the TTL")
|
||||
.expect("cached success must remain successful");
|
||||
let cached = fresh_cached_data_usage_snapshot(
|
||||
&cache,
|
||||
loaded_at + Duration::from_secs(1),
|
||||
Duration::from_secs(30),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("successful load must remain cached within the TTL")
|
||||
.expect("cached success must remain successful");
|
||||
assert_snapshot_bucket(&cached, "bucket");
|
||||
}
|
||||
|
||||
@@ -4633,6 +4680,7 @@ mod tests {
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
@@ -4666,6 +4714,7 @@ mod tests {
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
);
|
||||
|
||||
assert!(stale_result.is_none());
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
|
||||
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||
let cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info),
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
let stored = cache
|
||||
.as_ref()
|
||||
.expect("cached snapshot")
|
||||
.info
|
||||
.as_ref()
|
||||
.expect("known snapshot");
|
||||
let ttl = Duration::from_secs(30);
|
||||
|
||||
for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
|
||||
let value = fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, |info| {
|
||||
assert!(std::ptr::eq(info, stored), "projection must borrow the cached snapshot");
|
||||
info.buckets_usage.get(bucket).map(|usage| usage.size)
|
||||
})
|
||||
.expect("fresh snapshot")
|
||||
.expect("successful snapshot");
|
||||
let owned = fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, DataUsageInfo::clone)
|
||||
.expect("fresh snapshot")
|
||||
.expect("successful snapshot");
|
||||
assert_eq!(value, expected);
|
||||
assert_eq!(value, owned.buckets_usage.get(bucket).map(|usage| usage.size));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let ttl = Duration::from_secs(30);
|
||||
let never_project = |_: &DataUsageInfo| -> u64 { panic!("unusable snapshots must not be projected") };
|
||||
assert!(fresh_cached_data_usage_snapshot(&None, loaded_at, ttl, never_project).is_none());
|
||||
|
||||
let mut cache = None;
|
||||
let failure = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, 1, 1, never_project)
|
||||
.expect("matching generation");
|
||||
assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
|
||||
assert!(
|
||||
fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, never_project)
|
||||
.expect("negative cache remains fresh")
|
||||
.is_err()
|
||||
);
|
||||
|
||||
cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
assert!(fresh_cached_data_usage_snapshot(&cache, loaded_at + ttl, ttl, never_project).is_none());
|
||||
cache = None;
|
||||
let rejected = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(LoadedUsageBaseline {
|
||||
info: data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH),
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
}),
|
||||
loaded_at,
|
||||
1,
|
||||
2,
|
||||
never_project,
|
||||
);
|
||||
assert!(rejected.is_none());
|
||||
assert!(cache.is_none(), "rejected generation must not republish stale data");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projected_publication_moves_bucket_maps_into_cache() {
|
||||
let info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
|
||||
let original_key = info.buckets_usage.get_key_value("known").expect("source bucket").0.as_ptr();
|
||||
let mut cache = None;
|
||||
let usage = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(LoadedUsageBaseline {
|
||||
info,
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
}),
|
||||
tokio::time::Instant::now(),
|
||||
1,
|
||||
1,
|
||||
|info| {
|
||||
assert_eq!(
|
||||
info.buckets_usage
|
||||
.get_key_value("known")
|
||||
.expect("projected bucket")
|
||||
.0
|
||||
.as_ptr(),
|
||||
original_key
|
||||
);
|
||||
info.buckets_usage.get("known").map(|usage| usage.size)
|
||||
},
|
||||
)
|
||||
.expect("matching generation")
|
||||
.expect("successful publication");
|
||||
assert_eq!(usage, Some(42));
|
||||
let cached = cache
|
||||
.as_ref()
|
||||
.expect("published cache")
|
||||
.info
|
||||
.as_ref()
|
||||
.expect("published snapshot");
|
||||
assert_eq!(
|
||||
cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
|
||||
original_key
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
|
||||
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||
use crate::config::com::save_config;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
|
||||
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||
info.bucket_sizes.insert("empty".to_string(), 0);
|
||||
info.buckets_count = 2;
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&info).expect("encode snapshot"),
|
||||
)
|
||||
.await
|
||||
.expect("persist snapshot");
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
|
||||
for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
|
||||
assert_eq!(persisted_bucket_usage(store.clone(), bucket).await, expected);
|
||||
let owned = load_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("owned cached snapshot");
|
||||
assert_eq!(expected, owned.buckets_usage.get(bucket).map(|usage| usage.size));
|
||||
}
|
||||
info.buckets_usage.get_mut("known").expect("known bucket").size = 84;
|
||||
info.bucket_sizes.insert("known".to_string(), 84);
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&info).expect("encode new snapshot"),
|
||||
)
|
||||
.await
|
||||
.expect("persist new snapshot without invalidating cache");
|
||||
assert_eq!(
|
||||
persisted_bucket_usage(store.clone(), "known").await,
|
||||
Some(42),
|
||||
"warm reads reuse the TTL cache"
|
||||
);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
assert_eq!(
|
||||
persisted_bucket_usage(store.clone(), "known").await,
|
||||
Some(84),
|
||||
"invalidation exposes the new snapshot"
|
||||
);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
if let Some(cancel) = store.ctx.background_cancel_token() {
|
||||
cancel.cancel();
|
||||
}
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
disk.close().await.expect("stop fixture disk monitoring");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -159,10 +159,6 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
std::fs::symlink_metadata(path)
|
||||
}
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir_all(path.as_ref()).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
e.raw_os_error() == Some(libc::EISDIR)
|
||||
|| e.kind() == io::ErrorKind::IsADirectory
|
||||
@@ -210,10 +206,6 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir(path.as_ref()).await
|
||||
}
|
||||
|
||||
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
@@ -472,17 +464,6 @@ mod tests {
|
||||
assert!(metadata.file_type().is_symlink());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_make_dir_all() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let nested_path = temp_dir.path().join("level1").join("level2").join("level3");
|
||||
|
||||
// Should create nested directories
|
||||
assert!(make_dir_all(&nested_path).await.is_ok());
|
||||
assert!(nested_path.exists());
|
||||
assert!(nested_path.is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remove_file() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
@@ -555,17 +536,6 @@ mod tests {
|
||||
assert!(!dir_path.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mkdir() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let dir_path = temp_dir.path().join("test_mkdir");
|
||||
|
||||
// Create directory
|
||||
assert!(mkdir(&dir_path).await.is_ok());
|
||||
assert!(dir_path.exists());
|
||||
assert!(dir_path.is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rename() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
+10272
-1095
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,740 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::error::{DiskError, Result};
|
||||
use std::{fs::File, sync::Arc};
|
||||
|
||||
/// A target disk's admitted artifact publication. Physical mutation workers
|
||||
/// retain this Arc until their final syscall and rollback have finished.
|
||||
pub struct ArtifactLease {
|
||||
_active: File,
|
||||
_key_directory: File,
|
||||
_seal_directory: File,
|
||||
_root: File,
|
||||
}
|
||||
|
||||
pub type ArtifactPutLease = ArtifactLease;
|
||||
pub type ArtifactRepairLease = ArtifactLease;
|
||||
|
||||
/// Read-only census owns both already-closed physical mutation domains.
|
||||
pub(super) struct ArtifactInventoryLease {
|
||||
_put: Arc<ArtifactLease>,
|
||||
_repair: Arc<ArtifactLease>,
|
||||
}
|
||||
|
||||
pub(super) async fn drain_closed(root: File, key: [u8; 32], deadline: tokio::time::Instant) -> Result<ArtifactInventoryLease> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::drain_closed(root, key, deadline).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, deadline);
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum Domain {
|
||||
Put,
|
||||
Repair,
|
||||
}
|
||||
|
||||
impl Domain {
|
||||
fn directory(self) -> &'static str {
|
||||
match self {
|
||||
Self::Put => ".rustfs-artifact-put-seals",
|
||||
Self::Repair => ".rustfs-artifact-repair-seals",
|
||||
}
|
||||
}
|
||||
|
||||
fn closed(self) -> &'static [u8] {
|
||||
match self {
|
||||
Self::Put => b"rustfs-artifact-put-sealed-v1\n",
|
||||
Self::Repair => b"rustfs-artifact-repair-sealed-v1\n",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pinned_root(disk: &super::LocalDisk) -> Result<File> {
|
||||
#[cfg(target_os = "linux")]
|
||||
let root = disk.mount_lease.clone();
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
let root = disk.artifact_identity_root.clone();
|
||||
#[cfg(not(unix))]
|
||||
let _ = disk;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
root.try_clone().map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
|
||||
pub(super) async fn acquire(root: File, key: [u8; 32]) -> Result<Arc<ArtifactPutLease>> {
|
||||
acquire_in(root, key, Domain::Put).await
|
||||
}
|
||||
|
||||
pub(super) async fn acquire_in(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::acquire(root, key, domain).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, domain);
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn seal(root: File, key: [u8; 32]) -> Result<()> {
|
||||
seal_in(root, key, Domain::Put).await
|
||||
}
|
||||
|
||||
pub(super) async fn seal_in(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::seal(root, key, domain).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, domain);
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
mod unix {
|
||||
use super::*;
|
||||
use rustix::fs::{FlockOperation, Mode, OFlags, flock, mkdirat, openat};
|
||||
use rustix::io::Errno;
|
||||
use std::{io, io::Read, io::Write, os::unix::fs::MetadataExt, time::Duration};
|
||||
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(10);
|
||||
const LOCK_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct Opened {
|
||||
root: File,
|
||||
seals: File,
|
||||
key: File,
|
||||
admission: File,
|
||||
active: File,
|
||||
domain: Domain,
|
||||
}
|
||||
|
||||
fn invalid() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "invalid artifact seal filesystem state")
|
||||
}
|
||||
|
||||
fn directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
match mkdirat(parent, name, Mode::RWXU) {
|
||||
Ok(()) | Err(Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_dir() || metadata.dev() != device || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn regular(file: &File, device: u64) -> io::Result<std::fs::Metadata> {
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() || metadata.dev() != device || metadata.nlink() != 1 || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
fn lock_file(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::CREATE | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)?);
|
||||
if regular(&file, device)?.len() != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
impl Opened {
|
||||
fn open(root: File, key: [u8; 32], domain: Domain) -> io::Result<Self> {
|
||||
let metadata = root.metadata()?;
|
||||
if !metadata.is_dir() {
|
||||
return Err(invalid());
|
||||
}
|
||||
let seals = directory(&root, domain.directory(), metadata.dev())?;
|
||||
let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
|
||||
let key = directory(&seals, &name, metadata.dev())?;
|
||||
// Neither lock inode is ever removed or replaced, including after
|
||||
// closing. Independent processes must always lock the same inode.
|
||||
let admission = lock_file(&key, "admission.lock", metadata.dev())?;
|
||||
let active = lock_file(&key, "active.lock", metadata.dev())?;
|
||||
Ok(Self {
|
||||
root,
|
||||
seals,
|
||||
key,
|
||||
admission,
|
||||
active,
|
||||
domain,
|
||||
})
|
||||
}
|
||||
|
||||
fn closed(&self) -> io::Result<Option<File>> {
|
||||
self.closed_with_flags(OFlags::RDWR)
|
||||
}
|
||||
|
||||
fn closed_with_flags(&self, access: OFlags) -> io::Result<Option<File>> {
|
||||
let mut file = match openat(
|
||||
&self.key,
|
||||
"closed",
|
||||
access | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::empty(),
|
||||
) {
|
||||
Ok(file) => File::from(file),
|
||||
Err(Errno::NOENT) => return Ok(None),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let length = u64::try_from(self.domain.closed().len()).map_err(|_| invalid())?;
|
||||
if regular(&file, self.root.metadata()?.dev())?.len() != length {
|
||||
return Err(invalid());
|
||||
}
|
||||
let mut bytes = vec![0; self.domain.closed().len()];
|
||||
file.read_exact(&mut bytes)?;
|
||||
if bytes != self.domain.closed() {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(Some(file))
|
||||
}
|
||||
|
||||
fn admit(self) -> io::Result<Arc<ArtifactLease>> {
|
||||
if self.closed()?.is_some() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::PermissionDenied,
|
||||
"artifact attempt mutation domain is closed",
|
||||
));
|
||||
}
|
||||
// Admission serializes the closed check with taking this shared
|
||||
// physical owner. A sealer cannot pass between those two steps.
|
||||
flock(&self.active, FlockOperation::NonBlockingLockShared)?;
|
||||
flock(&self.admission, FlockOperation::Unlock)?;
|
||||
Ok(Arc::new(ArtifactLease {
|
||||
_active: self.active,
|
||||
_key_directory: self.key,
|
||||
_seal_directory: self.seals,
|
||||
_root: self.root,
|
||||
}))
|
||||
}
|
||||
|
||||
fn close(self) -> io::Result<Self> {
|
||||
let marker = match self.closed()? {
|
||||
Some(marker) => marker,
|
||||
None => {
|
||||
let mut marker = File::from(openat(
|
||||
&self.key,
|
||||
"closed",
|
||||
OFlags::CREATE | OFlags::EXCL | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)?);
|
||||
regular(&marker, self.root.metadata()?.dev())?;
|
||||
marker.write_all(self.domain.closed())?;
|
||||
marker
|
||||
}
|
||||
};
|
||||
// No admission is released until the permanent denial and every
|
||||
// ancestor entry are durable. A failed/cancelled close never removes it.
|
||||
marker.sync_all()?;
|
||||
self.admission.sync_all()?;
|
||||
self.active.sync_all()?;
|
||||
self.key.sync_all()?;
|
||||
self.seals.sync_all()?;
|
||||
self.root.sync_all()?;
|
||||
flock(&self.admission, FlockOperation::Unlock)?;
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
async fn lock(file: &File, operation: FlockOperation) -> io::Result<()> {
|
||||
tokio::time::timeout(LOCK_TIMEOUT, async {
|
||||
loop {
|
||||
match flock(file, operation) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(Errno::WOULDBLOCK | Errno::INTR) => tokio::time::sleep(POLL_INTERVAL).await,
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "artifact disk lease lock deadline"))?
|
||||
}
|
||||
|
||||
async fn open(root: File, key: [u8; 32], domain: Domain) -> Result<Opened> {
|
||||
tokio::task::spawn_blocking(move || Opened::open(root, key, domain))
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
fn read_error(error: io::Error) -> DiskError {
|
||||
match error.kind() {
|
||||
io::ErrorKind::NotFound => DiskError::other("artifact inventory requires closed PUT and repair domains"),
|
||||
io::ErrorKind::InvalidData | io::ErrorKind::NotADirectory => DiskError::FileCorrupt,
|
||||
io::ErrorKind::TimedOut => DiskError::Timeout,
|
||||
_ if error.raw_os_error() == Some(Errno::LOOP.raw_os_error()) => DiskError::FileCorrupt,
|
||||
_ => DiskError::from(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn existing_directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if metadata.dev() != device || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn existing(root: File, key: [u8; 32], domain: Domain) -> io::Result<Opened> {
|
||||
let device = root.metadata()?.dev();
|
||||
let seals = existing_directory(&root, domain.directory(), device)?;
|
||||
let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
|
||||
let key = existing_directory(&seals, &name, device)?;
|
||||
let lock = |name| -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
&key,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
if regular(&file, device)?.len() != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
};
|
||||
let admission = lock("admission.lock")?;
|
||||
let active = lock("active.lock")?;
|
||||
let opened = Opened {
|
||||
root,
|
||||
seals,
|
||||
key,
|
||||
admission,
|
||||
active,
|
||||
domain,
|
||||
};
|
||||
if opened.closed_with_flags(OFlags::RDONLY)?.is_none() {
|
||||
return Err(io::Error::from(io::ErrorKind::NotFound));
|
||||
}
|
||||
Ok(opened)
|
||||
}
|
||||
|
||||
fn retained(opened: Opened) -> Arc<ArtifactLease> {
|
||||
Arc::new(ArtifactLease {
|
||||
_active: opened.active,
|
||||
_key_directory: opened.key,
|
||||
_seal_directory: opened.seals,
|
||||
_root: opened.root,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn drain_closed(
|
||||
root: File,
|
||||
key: [u8; 32],
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<ArtifactInventoryLease> {
|
||||
let (put, repair) = tokio::task::spawn_blocking(move || {
|
||||
let put = existing(root.try_clone()?, key, Domain::Put)?;
|
||||
let repair = existing(root, key, Domain::Repair)?;
|
||||
Ok::<_, io::Error>((put, repair))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(read_error)?;
|
||||
// Lock order: closed PUT active exclusive -> closed repair active
|
||||
// exclusive. No admission lock is acquired and no seal is created.
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
lock(&put.active, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
lock(&repair.active, FlockOperation::NonBlockingLockExclusive).await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
.map_err(read_error)?;
|
||||
Ok(ArtifactInventoryLease {
|
||||
_put: retained(put),
|
||||
_repair: retained(repair),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn acquire(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
|
||||
let opened = open(root, key, domain).await?;
|
||||
// Lock order: admission exclusive -> active shared. Contention sleeps
|
||||
// asynchronously; no blocking executor thread waits for another owner.
|
||||
lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
tokio::task::spawn_blocking(move || opened.admit())
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
pub(super) async fn seal(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
|
||||
let opened = open(root, key, domain).await?;
|
||||
lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
let opened = tokio::task::spawn_blocking(move || opened.close())
|
||||
.await
|
||||
.map_err(DiskError::from)??;
|
||||
// Closing is already durable. Cancellation or timeout here only loses
|
||||
// the confirmation; admission stays permanently closed across reopen.
|
||||
lock(&opened.active, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
const KEY: [u8; 32] = [7; 32];
|
||||
|
||||
fn root(path: &Path) -> File {
|
||||
use rustix::fs::{Mode, OFlags, open};
|
||||
File::from(
|
||||
open(
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)
|
||||
.expect("open pinned test root"),
|
||||
)
|
||||
}
|
||||
|
||||
fn directory(path: &Path) -> std::path::PathBuf {
|
||||
path.join(Domain::Put.directory())
|
||||
.join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
|
||||
}
|
||||
|
||||
async fn closed(path: &Path) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !directory(path).join("closed").exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("sealer reaches its monotonic close point");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_closing_domain_is_independent_and_survives_reopen() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
seal(root(temp.path()), KEY).await.expect("PUT closes first");
|
||||
let repair = acquire_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("PUT closure permits existing-version repair");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err());
|
||||
drop(repair);
|
||||
seal_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("repair closes independently");
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
seal_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("reopened repair close is durable");
|
||||
|
||||
let other = [8; 32];
|
||||
seal_in(root(temp.path()), other, Domain::Repair)
|
||||
.await
|
||||
.expect("close only repair admission");
|
||||
drop(
|
||||
acquire(root(temp.path()), other)
|
||||
.await
|
||||
.expect("repair closure permits PUT domain"),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_cancelled_waiter_keeps_physical_owner_and_durable_close() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("repair admitted");
|
||||
let (entered, started) = tokio::sync::oneshot::channel();
|
||||
let (release, waiting) = std::sync::mpsc::channel();
|
||||
let owner = tokio::spawn(async move {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = writer;
|
||||
let _ = entered.send(());
|
||||
let _ = waiting.recv();
|
||||
})
|
||||
.await
|
||||
.expect("repair worker");
|
||||
});
|
||||
started.await.expect("physical repair entered");
|
||||
owner.abort();
|
||||
assert!(owner.await.expect_err("cancel repair waiter").is_cancelled());
|
||||
let sealer = tokio::spawn(seal_in(root(temp.path()), KEY, Domain::Repair));
|
||||
let marker = temp
|
||||
.path()
|
||||
.join(Domain::Repair.directory())
|
||||
.join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
|
||||
.join("closed");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !marker.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("repair admission durably closes");
|
||||
assert!(!sealer.is_finished(), "ARTIFACT_REPAIR_CANCELLED_PHYSICAL_TAIL");
|
||||
sealer.abort();
|
||||
assert!(sealer.await.expect_err("cancel repair close waiter").is_cancelled());
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
let pending = seal_in(root(temp.path()), KEY, Domain::Repair);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_REPAIR_REOPEN_MUST_DRAIN_PHYSICAL_TAIL"
|
||||
);
|
||||
release.send(()).expect("release physical repair");
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical drain bounded")
|
||||
.expect("reopened close confirms repair drain");
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_seal_waits_for_independent_openers_and_last_clone() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let first = acquire(root(temp.path()), KEY).await.expect("first admitted writer");
|
||||
let tail = first.clone();
|
||||
let second = acquire(root(temp.path()), KEY).await.expect("independent shared writer");
|
||||
let pending = seal(root(temp.path()), KEY);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_ACTIVE_WRITERS_DRAIN"
|
||||
);
|
||||
drop(first);
|
||||
drop(second);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_PHYSICAL_CLONE_RETAINED"
|
||||
);
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSE_REJECTS_ADMISSION");
|
||||
drop(tail);
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical tail drain bounded")
|
||||
.expect("seal complete");
|
||||
seal(root(temp.path()), KEY)
|
||||
.await
|
||||
.expect("independent reopen preserves closed state");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSED_AFTER_REOPEN");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_close_cancellation_leaves_durable_denial() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("admitted writer");
|
||||
let sealer = tokio::spawn(seal(root(temp.path()), KEY));
|
||||
closed(temp.path()).await;
|
||||
sealer.abort();
|
||||
assert!(sealer.await.expect_err("cancel close waiter").is_cancelled());
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_CANCELLED_CLOSE_STAYS_CLOSED"
|
||||
);
|
||||
drop(writer);
|
||||
seal(root(temp.path()), KEY).await.expect("reopen finishes physical drain");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CANCELLED_CLOSE_REOPEN");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_physical_worker_retains_lease_after_waiter_cancellation() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("physical writer admitted");
|
||||
let (entered, started) = tokio::sync::oneshot::channel();
|
||||
let (release, waiting) = std::sync::mpsc::channel();
|
||||
let owner = tokio::spawn(async move {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _physical_owner = writer;
|
||||
let _ = entered.send(());
|
||||
let _ = waiting.recv();
|
||||
})
|
||||
.await
|
||||
.expect("physical worker");
|
||||
});
|
||||
started.await.expect("physical syscall entered");
|
||||
owner.abort();
|
||||
assert!(owner.await.expect_err("cancel async writer").is_cancelled());
|
||||
let pending = seal(root(temp.path()), KEY);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_CANCELLED_PHYSICAL_TAIL"
|
||||
);
|
||||
release.send(()).expect("release physical worker");
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical worker drains")
|
||||
.expect("seal confirms real completion");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_seal_waits_for_another_process() {
|
||||
const CHILD_ROOT: &str = "RUSTFS_TEST_ARTIFACT_PUT_SEAL_ROOT";
|
||||
if let Some(path) = std::env::var_os(CHILD_ROOT) {
|
||||
seal(root(Path::new(&path)), KEY)
|
||||
.await
|
||||
.expect("child closes and drains parent writer");
|
||||
assert!(acquire(root(Path::new(&path)), KEY).await.is_err(), "ARTIFACT_PUT_CHILD_CLOSED");
|
||||
return;
|
||||
}
|
||||
struct Child(std::process::Child);
|
||||
impl Drop for Child {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("parent owns physical write");
|
||||
let mut child = Child(
|
||||
std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"disk::local::artifact_lease::tests::artifact_put_seal_waits_for_another_process",
|
||||
])
|
||||
.env(CHILD_ROOT, temp.path())
|
||||
.spawn()
|
||||
.expect("start independent sealer"),
|
||||
);
|
||||
closed(temp.path()).await;
|
||||
assert!(child.0.try_wait().expect("probe child").is_none(), "ARTIFACT_PUT_CROSS_PROCESS_DRAIN");
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_CROSS_PROCESS_ADMISSION_CLOSED"
|
||||
);
|
||||
drop(writer);
|
||||
let status = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let Some(status) = child.0.try_wait().expect("child wait") {
|
||||
break status;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("child physically drains");
|
||||
assert!(status.success(), "ARTIFACT_PUT_CROSS_PROCESS_SEAL");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_PROCESS_REOPEN_CLOSED");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_rejects_symlinks_at_every_seal_component() {
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
for component in ["directory", "key", "admission.lock", "active.lock", "closed"] {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let outside = tempfile::tempdir().expect("symlink destination");
|
||||
let sentinel = outside.path().join("untouched");
|
||||
std::fs::write(&sentinel, b"unchanged").expect("sentinel");
|
||||
let target = match component {
|
||||
"directory" => temp.path().join(Domain::Put.directory()),
|
||||
"key" => {
|
||||
std::fs::create_dir(temp.path().join(Domain::Put.directory())).expect("seal directory");
|
||||
std::fs::set_permissions(temp.path().join(Domain::Put.directory()), std::fs::Permissions::from_mode(0o700))
|
||||
.expect("private seal directory");
|
||||
directory(temp.path())
|
||||
}
|
||||
file => {
|
||||
drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
|
||||
let target = directory(temp.path()).join(file);
|
||||
if file != "closed" {
|
||||
std::fs::remove_file(&target).expect("replace fixture lock");
|
||||
}
|
||||
target
|
||||
}
|
||||
};
|
||||
symlink(
|
||||
if matches!(component, "directory" | "key") {
|
||||
outside.path()
|
||||
} else {
|
||||
&sentinel
|
||||
},
|
||||
target,
|
||||
)
|
||||
.expect("symlink fixture");
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_SYMLINK_REJECTED: {component}"
|
||||
);
|
||||
assert!(
|
||||
seal(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_SYMLINK_SEAL_REJECTED: {component}"
|
||||
);
|
||||
assert_eq!(std::fs::read(&sentinel).expect("sentinel intact"), b"unchanged");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_rejects_corrupt_closed_marker() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
|
||||
let marker = directory(temp.path()).join("closed");
|
||||
std::fs::write(&marker, b"partial").expect("corrupt marker fixture");
|
||||
std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o600)).expect("marker permissions");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_FAILS_CLOSED");
|
||||
assert!(seal(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_NOT_CONFIRMED");
|
||||
assert_eq!(std::fs::read(marker).expect("corruption retained"), b"partial");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_stays_on_pinned_root_after_path_replacement() {
|
||||
let temp = tempfile::tempdir().expect("disk root parent");
|
||||
let current = temp.path().join("current");
|
||||
let original = temp.path().join("original");
|
||||
std::fs::create_dir(¤t).expect("initial disk root");
|
||||
let pin = root(¤t);
|
||||
drop(
|
||||
acquire(pin.try_clone().expect("root pin clone"), KEY)
|
||||
.await
|
||||
.expect("initialize pinned seal"),
|
||||
);
|
||||
std::fs::rename(¤t, &original).expect("replace configured pathname");
|
||||
std::fs::create_dir(¤t).expect("replacement disk root");
|
||||
seal(pin, KEY).await.expect("close original disk through its descriptor");
|
||||
assert!(directory(&original).join("closed").exists(), "ARTIFACT_PUT_ROOT_PINNED");
|
||||
assert!(!current.join(Domain::Put.directory()).exists(), "ARTIFACT_PUT_REPLACEMENT_UNTOUCHED");
|
||||
assert!(acquire(root(&original), KEY).await.is_err());
|
||||
}
|
||||
}
|
||||
@@ -345,6 +345,8 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
let publication_root = publication_root.clone();
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
if create_source_parent {
|
||||
#[cfg(test)]
|
||||
super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
|
||||
std::fs::create_dir_all(&source_parent)?;
|
||||
}
|
||||
let guard = match os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root) {
|
||||
@@ -365,9 +367,15 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
};
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let result = {
|
||||
let _ = mutation_lease;
|
||||
if create_source_parent {
|
||||
fs::create_dir_all(source_parent).await.map_err(to_file_error)?;
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
|
||||
std::fs::create_dir_all(source_parent)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
let guard =
|
||||
os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root).map_err(|err| {
|
||||
@@ -398,30 +406,31 @@ pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
prepared: Option<Arc<os::NamespaceMutationLease>>,
|
||||
}
|
||||
|
||||
pub(in crate::disk) struct PreparedRenameData {
|
||||
file_info: FileInfo,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
async fn prepare_rename_data_admission(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
source_volume: Option<&str>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<PreparedRenameData> {
|
||||
let (src_volume, src_path) = source;
|
||||
let (dst_volume, dst_path) = destination;
|
||||
let mut fi = fi;
|
||||
// A non-force DeleteBucket must not remove a directory while a local
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
// optimistic; this lease establishes the local commit/delete order and
|
||||
// remains owned by any blocking syscall that outlives async cancellation.
|
||||
let (dst_volume_dir, destination_object_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
|
||||
let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
let quota_fence_token =
|
||||
match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) {
|
||||
Some(value) => {
|
||||
@@ -455,16 +464,27 @@ impl LocalDisk {
|
||||
.strip_prefix(self.io_root())
|
||||
.map_err(|_| DiskError::InvalidPath)?,
|
||||
);
|
||||
let metadata_lease =
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, state.namespace_owner.take()).await;
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await;
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, namespace_owner).await;
|
||||
let mutation_lease = if let Some(source_volume) = source_volume {
|
||||
os::acquire_rename_data_mutation_lease_for_volumes(
|
||||
&self.root,
|
||||
source_volume,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await
|
||||
};
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -472,6 +492,99 @@ impl LocalDisk {
|
||||
fi.erasure.index = 0;
|
||||
}
|
||||
fi.validate_for_metadata_read()?;
|
||||
Ok(PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn begin_legacy_rename_data(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
permit: crate::disk::io_admission::DiskIoPermit,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<PreparedRenameData> {
|
||||
// Capture the same permit, mount FD and listener-instance owner before
|
||||
// quota -> metadata -> namespace -> sorted volume READ admission.
|
||||
#[cfg(target_os = "linux")]
|
||||
let owner: Arc<dyn Send + Sync> = Arc::new((permit, self.mount_lease.clone(), namespace_owner));
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let owner: Arc<dyn Send + Sync> = Arc::new((permit, namespace_owner));
|
||||
let prepared = self
|
||||
.prepare_rename_data_admission(fi.clone(), source, destination, Some(source.0), Some(owner))
|
||||
.await?;
|
||||
for volume in [Some(source.0), (source.0 != destination.0).then_some(destination.0)]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if !skip_access_checks(volume) {
|
||||
let path = self.io_get_bucket_path(volume)?;
|
||||
let metadata = fs::symlink_metadata(path)
|
||||
.await
|
||||
.map_err(crate::disk::error_conv::to_volume_error)?;
|
||||
if !metadata.is_dir() {
|
||||
return Err(DiskError::VolumeNotFound);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_prepared(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
prepared: PreparedRenameData,
|
||||
) -> Result<RenameDataResp> {
|
||||
let PreparedRenameData {
|
||||
file_info,
|
||||
mutation_lease,
|
||||
} = prepared;
|
||||
let mut state = RenameDataState {
|
||||
prepared: Some(mutation_lease),
|
||||
..Default::default()
|
||||
};
|
||||
self.rename_data_inner(source.0, source.1, file_info, destination.0, destination.1, &mut state)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
} = if let Some(mutation_lease) = state.prepared.take() {
|
||||
PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
}
|
||||
} else {
|
||||
self.prepare_rename_data_admission(
|
||||
fi,
|
||||
(src_volume, src_path),
|
||||
(dst_volume, dst_path),
|
||||
None,
|
||||
state.namespace_owner.take(),
|
||||
)
|
||||
.await?
|
||||
};
|
||||
let no_inline = fi.data.is_none() && fi.size > 0;
|
||||
let fi_healing = fi.is_healing();
|
||||
// Snapshot the destination part paths before `fi` is consumed below. These
|
||||
// are the descriptors a reader may hold for the version this call is about
|
||||
// to replace (backlog#1145); readers build the identical string in
|
||||
@@ -507,6 +620,7 @@ impl LocalDisk {
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(dst_volume)
|
||||
&& let Err(e) = super::super::fs::access_std(&dst_volume_dir)
|
||||
{
|
||||
@@ -599,10 +713,10 @@ impl LocalDisk {
|
||||
|
||||
if preserve_upload_metadata {
|
||||
let source_object_path = self.get_object_path(src_volume, src_path)?;
|
||||
let authority_owner: Arc<dyn Send + Sync> = if source_object_path == root_destination_object_path {
|
||||
let authority_owner: Arc<dyn Send + Sync> = if source_object_path == self.get_object_path(dst_volume, dst_path)? {
|
||||
commit.mutation_lease.clone()
|
||||
} else {
|
||||
os::acquire_metadata_mutation_lease(&source_object_path, Some(commit.mutation_lease.clone())).await
|
||||
os::try_acquire_metadata_mutation_lease_with_owner(&source_object_path, commit.mutation_lease.clone())?
|
||||
};
|
||||
let authority = self.io_get_object_path(src_volume, &format!("{src_path}/{STORAGE_FORMAT_FILE}"))?;
|
||||
let recovery = self.io_get_object_path(
|
||||
@@ -610,14 +724,10 @@ impl LocalDisk {
|
||||
&format!("{src_path}/{}/{STORAGE_FORMAT_FILE}", super::super::MULTIPART_RECOVERY_DIR),
|
||||
)?;
|
||||
check_path_length(recovery.to_string_lossy().as_ref())?;
|
||||
os::rename_all_with_owner(
|
||||
&authority,
|
||||
&recovery,
|
||||
&src_volume_dir,
|
||||
commit.publication_root,
|
||||
Some(authority_owner.clone()),
|
||||
)
|
||||
.await?;
|
||||
// Both volume READs are already held. Neither source key may wait
|
||||
// behind a competing owner that is queued behind a volume WRITE.
|
||||
let recovery_lease = os::try_acquire_namespace_mutation_lease_with_owner(&recovery, authority_owner.clone())?;
|
||||
os::rename_all_with_lease(&authority, &recovery, &src_volume_dir, commit.publication_root, recovery_lease).await?;
|
||||
// This is system metadata authority, so use its existing strict
|
||||
// durability policy even when the object payload is relaxed.
|
||||
if effective_durability(src_volume).syncs_commit_metadata() {
|
||||
@@ -838,7 +948,7 @@ impl LocalDisk {
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
|
||||
let backup_parent = dst_file_parent.join(old_data_dir.to_string());
|
||||
#[cfg(not(windows))]
|
||||
if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await {
|
||||
if let Err(err) = os::make_dir_all_with_owner(&backup_parent, &skip_parent, commit.mutation_lease.clone()).await {
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1305,18 +1415,25 @@ impl LocalDisk {
|
||||
let backup_path = dst_parent
|
||||
.join(rollback_data_dir.to_string())
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
// rename_all acquires the backup path's namespace lease. Do not
|
||||
// hold a disk admission while acquiring another namespace lock.
|
||||
// Keep backup mutations serialized, but never wait for a second
|
||||
// namespace while holding the volume READ: its owner can be
|
||||
// queued behind a volume DELETE that is waiting for this commit.
|
||||
drop(file_sync_admission.take());
|
||||
if let Err(err) = os::rename_all_with_owner(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
commit.publication_root,
|
||||
Some(commit.mutation_lease.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let backup_result =
|
||||
match os::try_acquire_namespace_mutation_lease_with_owner(&backup_path, commit.mutation_lease.clone()) {
|
||||
Ok(lease) => {
|
||||
os::rename_all_with_lease(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
commit.publication_root,
|
||||
lease,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if let Err(err) = backup_result {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+154
-36
@@ -27,6 +27,7 @@ mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub use local::artifact_inventory::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
pub mod os;
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -73,6 +74,10 @@ use crate::cluster::rpc::RemoteDisk;
|
||||
use crate::cluster::rpc::build_internode_data_transport_from_env;
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::disk::disk_store::LocalDiskWrapper;
|
||||
pub use crate::disk::disk_store::{
|
||||
LegacyFileStream, LegacyFileWrite, LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete,
|
||||
LegacyPathDeleteOperation, LegacyRenameData, LegacyRenameFile,
|
||||
};
|
||||
use crate::disk::health_state::RuntimeDriveHealthState;
|
||||
use crate::disk::local::ScanGuard;
|
||||
use bytes::Bytes;
|
||||
@@ -93,6 +98,7 @@ use uuid::Uuid;
|
||||
/// Independent admission and physical ownership for one disk rename.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RenameDataGuards {
|
||||
pub(crate) artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
@@ -234,6 +240,117 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Capture local file-write admission before authoritative classification.
|
||||
pub async fn begin_legacy_file_write(&self, volume: &str) -> Result<LegacyFileWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_file_write(volume).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_file_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyFileWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_file_write_with_namespace_owner(volume, owner).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture metadata publication admission before authoritative classification.
|
||||
pub async fn begin_legacy_metadata_write(&self, volume: &str, path: &str) -> Result<LegacyMetadataWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_metadata_write(volume, path).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_metadata_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyMetadataWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_metadata_write_with_namespace_owner(volume, path, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_legacy_admission_failure(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
error: &DiskError,
|
||||
started: std::time::Instant,
|
||||
) {
|
||||
if let Self::Local(disk) = self {
|
||||
disk.record_legacy_admission_failure(operation, error, started);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_path_delete(
|
||||
&self,
|
||||
volume: &str,
|
||||
operation: LegacyPathDeleteOperation,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<LegacyPathDelete> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_path_delete(volume, operation, owner, deadline).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_part_mutation(
|
||||
&self,
|
||||
source: Option<(&str, &str)>,
|
||||
destination: (&str, &str),
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
operation: LegacyPartOperation,
|
||||
) -> Result<LegacyPartMutation> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_part_mutation(source, destination, namespace_owner, operation)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_rename_file(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
durable: bool,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyRenameFile> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_rename_file(source, destination, durable, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_rename_data(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyRenameData> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_rename_data(source, fi, destination, namespace_owner).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
/// A read-only probe; the physical delete repeats the same leaf checks.
|
||||
pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
|
||||
match self {
|
||||
@@ -1057,13 +1174,27 @@ impl Disk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
|
||||
return self
|
||||
.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await;
|
||||
let bucket_scope = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path);
|
||||
let repair_scope = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path);
|
||||
if let Some(scope) = &bucket_scope {
|
||||
scope.check()?;
|
||||
}
|
||||
if let Some(scope) = &repair_scope {
|
||||
scope.check().map_err(DiskError::other)?;
|
||||
}
|
||||
let repair_lease = if let (Self::Local(disk), Some(scope)) = (self, &repair_scope) {
|
||||
if let Some(identity) = scope.receiver_identity() {
|
||||
scope.check_version(fi.version_id).await.map_err(DiskError::other)?;
|
||||
Some(disk.acquire_artifact_repair_lease(identity.receiver_key).await?)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
scope.check()?;
|
||||
self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, Some(scope))
|
||||
let owner: Option<Arc<dyn Send + Sync>> = (bucket_scope.is_some() || repair_scope.is_some())
|
||||
.then(|| Arc::new((bucket_scope, repair_scope, repair_lease)) as Arc<dyn Send + Sync>);
|
||||
self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, owner)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1074,10 +1205,20 @@ impl Disk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: RenameDataGuards,
|
||||
mut guards: RenameDataGuards,
|
||||
) -> RenameDataObservation {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
if let Some(identity) = guards.artifact_put.take() {
|
||||
let lease = match local_disk.acquire_artifact_put_lease(identity.receiver_key).await {
|
||||
Ok(lease) => lease,
|
||||
Err(error) => return RenameDataObservation::unknown(Err(error)),
|
||||
};
|
||||
// Keep the existing timeout policy; physical workers own the
|
||||
// receiver lease even after the caller's disk deadline.
|
||||
guards.namespace_owner =
|
||||
Some(Arc::new((guards.external_guard.clone(), guards.namespace_owner.take(), lease)));
|
||||
}
|
||||
local_disk
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
@@ -1085,39 +1226,17 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
(src_volume, src_path),
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
(dst_volume, dst_path),
|
||||
guards.scanner_publication_lease_token,
|
||||
guards.artifact_put.map(|identity| identity.incarnation),
|
||||
)
|
||||
.await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence_and_guard(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
scanner_publication_lease_token,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn rename_data_borrowed_with_fence_and_guard(
|
||||
&self,
|
||||
@@ -1138,12 +1257,11 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
(src_volume, src_path),
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
(dst_volume, dst_path),
|
||||
scanner_publication_lease_token,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
+553
-52
@@ -29,7 +29,8 @@ use std::{
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio::sync::{
|
||||
Mutex as AsyncMutex, OwnedMutexGuard, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit, oneshot,
|
||||
Mutex as AsyncMutex, OwnedMutexGuard, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, OwnedSemaphorePermit, RwLock, Semaphore,
|
||||
SemaphorePermit, oneshot,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
@@ -273,10 +274,41 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
RemoveComplete,
|
||||
StreamOpen,
|
||||
StreamOpenComplete,
|
||||
StreamWrite,
|
||||
StreamWriteComplete,
|
||||
StreamFlush,
|
||||
StreamFlushComplete,
|
||||
StreamShutdown,
|
||||
StreamShutdownComplete,
|
||||
#[cfg(test)]
|
||||
StreamDrainWaiting,
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
DirectOpened,
|
||||
#[cfg(test)]
|
||||
Rollback,
|
||||
#[cfg(test)]
|
||||
DirFsync,
|
||||
#[cfg(test)]
|
||||
FileSync,
|
||||
#[cfg(test)]
|
||||
PartPrepare,
|
||||
#[cfg(test)]
|
||||
PartRestore,
|
||||
#[cfg(test)]
|
||||
PartCleanup,
|
||||
#[cfg(test)]
|
||||
PartSettle,
|
||||
#[cfg(test)]
|
||||
PartSettleTimeout,
|
||||
#[cfg(test)]
|
||||
PartSettlePlanned,
|
||||
#[cfg(test)]
|
||||
PartSettleComplete,
|
||||
#[cfg(test)]
|
||||
NamespaceWaiting,
|
||||
}
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
@@ -357,6 +389,8 @@ pub struct LocalPublicationPause {
|
||||
_hook: prepared_publication_test_hooks::Guard,
|
||||
entered: oneshot::Receiver<()>,
|
||||
_release: std::sync::mpsc::Sender<()>,
|
||||
_completion_hook: Option<prepared_publication_test_hooks::Guard>,
|
||||
completed: Option<oneshot::Receiver<()>>,
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
@@ -365,6 +399,8 @@ pub enum LocalPublicationStage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
StreamWrite,
|
||||
StreamShutdown,
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
@@ -377,7 +413,26 @@ impl LocalPublicationPause {
|
||||
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
|
||||
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
|
||||
LocalPublicationStage::StreamWrite => prepared_publication_test_hooks::Stage::StreamWrite,
|
||||
LocalPublicationStage::StreamShutdown => prepared_publication_test_hooks::Stage::StreamShutdown,
|
||||
};
|
||||
let complete_stage = match stage {
|
||||
prepared_publication_test_hooks::Stage::StreamWrite => {
|
||||
Some(prepared_publication_test_hooks::Stage::StreamWriteComplete)
|
||||
}
|
||||
prepared_publication_test_hooks::Stage::StreamShutdown => {
|
||||
Some(prepared_publication_test_hooks::Stage::StreamShutdownComplete)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let completion = complete_stage.map(|stage| {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
|
||||
let _ = tx.send(());
|
||||
});
|
||||
(hook, rx)
|
||||
});
|
||||
let (completion_hook, completed) = completion.map_or((None, None), |(hook, rx)| (Some(hook), Some(rx)));
|
||||
let (entered_tx, entered) = oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
|
||||
@@ -388,9 +443,27 @@ impl LocalPublicationPause {
|
||||
_hook: hook,
|
||||
entered,
|
||||
_release: release,
|
||||
_completion_hook: completion_hook,
|
||||
completed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Release a stream barrier and observe its actual physical operation finish.
|
||||
pub async fn release_and_wait(self) -> std::result::Result<(), oneshot::error::RecvError> {
|
||||
let Self {
|
||||
_hook,
|
||||
entered: _,
|
||||
_release,
|
||||
_completion_hook,
|
||||
completed,
|
||||
} = self;
|
||||
drop(_release);
|
||||
if let Some(completed) = completed {
|
||||
completed.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn entered(&mut self) -> std::result::Result<(), oneshot::error::RecvError> {
|
||||
(&mut self.entered).await
|
||||
}
|
||||
@@ -1614,7 +1687,7 @@ fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'stati
|
||||
}
|
||||
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
type NamespaceMutationLock = AsyncMutex<()>;
|
||||
type NamespaceMutationLock = RwLock<()>;
|
||||
type NamespaceMutationLockRegistry = HashMap<PathBuf, Weak<NamespaceMutationLock>>;
|
||||
static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
@@ -1683,7 +1756,7 @@ impl DiskBlockingLane {
|
||||
struct DiskBlockingQueueState {
|
||||
jobs: VecDeque<Box<dyn FnOnce() + Send>>,
|
||||
active_workers: usize,
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
paused_workers: usize,
|
||||
}
|
||||
|
||||
@@ -1713,7 +1786,7 @@ struct DiskBlockingWorkerGuard {
|
||||
|
||||
impl Drop for DiskBlockingWorkerGuard {
|
||||
fn drop(&mut self) {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
if self.state != DiskBlockingWorkerState::Submitted {
|
||||
CURRENT_DISK_BLOCKING_QUEUE.with(|slot| {
|
||||
let _ = slot.borrow_mut().take();
|
||||
@@ -1772,11 +1845,11 @@ impl DiskBlockingQueue {
|
||||
|
||||
#[inline]
|
||||
fn effective_max_workers(&self, state: &DiskBlockingQueueState) -> usize {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
{
|
||||
MAX_NAMESPACE_WORKERS_PER_DISK.saturating_add(state.paused_workers)
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
#[cfg(not(any(test, feature = "test-util", feature = "e2e-test-hooks")))]
|
||||
{
|
||||
let _ = state;
|
||||
MAX_NAMESPACE_WORKERS_PER_DISK
|
||||
@@ -1828,7 +1901,7 @@ impl DiskBlockingQueue {
|
||||
};
|
||||
let worker = move || {
|
||||
let mut guard = guard;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
CURRENT_DISK_BLOCKING_QUEUE.with(|slot| {
|
||||
*slot.borrow_mut() = Some((Arc::clone(&queue), lane));
|
||||
});
|
||||
@@ -1925,18 +1998,18 @@ impl DiskBlockingQueue {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
thread_local! {
|
||||
static CURRENT_DISK_BLOCKING_QUEUE: std::cell::RefCell<Option<(Arc<DiskBlockingQueue>, DiskBlockingLane)>> =
|
||||
const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
pub(crate) struct BlockingTestHookGuard {
|
||||
queue: Option<(Arc<DiskBlockingQueue>, DiskBlockingLane)>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
impl Drop for BlockingTestHookGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some((queue, lane)) = self.queue.take() {
|
||||
@@ -1946,7 +2019,7 @@ impl Drop for BlockingTestHookGuard {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(any(test, feature = "test-util", feature = "e2e-test-hooks"))]
|
||||
pub(crate) fn enter_blocking_test_hook() -> BlockingTestHookGuard {
|
||||
let queue = CURRENT_DISK_BLOCKING_QUEUE.with(|slot| slot.borrow().clone());
|
||||
if let Some((ref queue, lane)) = queue {
|
||||
@@ -2033,12 +2106,12 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
|
||||
return lock;
|
||||
}
|
||||
|
||||
let lock = Arc::new(AsyncMutex::new(()));
|
||||
let lock = Arc::new(RwLock::new(()));
|
||||
locks.insert(path.to_path_buf(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
|
||||
static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
|
||||
static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<AsyncMutex<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
/// Serializes the complete xl.meta read/modify/commit transaction. This domain
|
||||
@@ -2049,21 +2122,36 @@ pub(crate) struct MetadataMutationLease {
|
||||
_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
fn disk_metadata_mutation_lock(object: &Path) -> Arc<AsyncMutex<()>> {
|
||||
let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
|
||||
locks.retain(|_, lock| lock.strong_count() > 0);
|
||||
if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
|
||||
lock
|
||||
} else {
|
||||
let lock = Arc::new(AsyncMutex::new(()));
|
||||
locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::disk) fn try_acquire_metadata_mutation_lease_with_owner(
|
||||
object: &Path,
|
||||
owner: Arc<dyn Send + Sync>,
|
||||
) -> Result<Arc<MetadataMutationLease>> {
|
||||
let guard = disk_metadata_mutation_lock(object)
|
||||
.try_lock_owned()
|
||||
.map_err(|_| DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock)))?;
|
||||
Ok(Arc::new(MetadataMutationLease {
|
||||
_guard: guard,
|
||||
_owner: Some(owner),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_metadata_mutation_lease(
|
||||
object: &Path,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<MetadataMutationLease> {
|
||||
let lock = {
|
||||
let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
|
||||
locks.retain(|_, lock| lock.strong_count() > 0);
|
||||
if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
|
||||
lock
|
||||
} else {
|
||||
let lock = Arc::new(AsyncMutex::new(()));
|
||||
locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
};
|
||||
let lock = disk_metadata_mutation_lock(object);
|
||||
Arc::new(MetadataMutationLease {
|
||||
_guard: lock.lock_owned().await,
|
||||
_owner: owner,
|
||||
@@ -2073,9 +2161,10 @@ pub(crate) async fn acquire_metadata_mutation_lease(
|
||||
/// Keeps a namespace transaction serialized even when its async waiter is
|
||||
/// cancelled while a blocking filesystem call is still running.
|
||||
pub(crate) struct NamespaceMutationLease {
|
||||
_namespace_guard: OwnedMutexGuard<()>,
|
||||
_staging_guard: Option<OwnedMutexGuard<()>>,
|
||||
_namespace_guard: OwnedRwLockWriteGuard<()>,
|
||||
_staging_guard: Option<OwnedRwLockWriteGuard<()>>,
|
||||
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||
_secondary_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
|
||||
// Independent of the quota claim; both survive cancellation of the waiter.
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
@@ -2098,15 +2187,44 @@ pub(in crate::disk) async fn acquire_namespace_mutation_lease_with_owner(
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
let blocking_queue = disk_blocking_queue_for_path(path);
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
|
||||
_namespace_guard: disk_namespace_mutation_lock(path).write_owned().await,
|
||||
_staging_guard: None,
|
||||
_volume_guard: None,
|
||||
_secondary_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
blocking_queue,
|
||||
})
|
||||
}
|
||||
|
||||
/// A transaction holding a volume READ must not queue for another namespace:
|
||||
/// its owner may already be waiting behind a queued volume WRITE.
|
||||
pub(in crate::disk) fn try_acquire_namespace_mutation_lease_with_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Arc<dyn Send + Sync>,
|
||||
) -> Result<Arc<NamespaceMutationLease>> {
|
||||
let guard = disk_namespace_mutation_lock(path)
|
||||
.try_write_owned()
|
||||
.map_err(|_| DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock)))?;
|
||||
Ok(Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: guard,
|
||||
_staging_guard: None,
|
||||
_volume_guard: None,
|
||||
_secondary_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: Some(namespace_owner),
|
||||
blocking_queue: disk_blocking_queue_for_path(path),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Protect a parent directory against replacement while sibling parts publish.
|
||||
/// Like exact-path acquisition, this must not wait while holding a volume READ.
|
||||
pub(in crate::disk) fn try_acquire_namespace_parent_read(path: &Path) -> Result<OwnedRwLockReadGuard<()>> {
|
||||
disk_namespace_mutation_lock(path)
|
||||
.try_read_owned()
|
||||
.map_err(|_| DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock)))
|
||||
}
|
||||
|
||||
/// Acquire object serialization before the volume read lock. Bucket deletion
|
||||
/// only acquires the volume write lock, so this order cannot form a lock cycle.
|
||||
pub(crate) async fn acquire_rename_data_mutation_lease(
|
||||
@@ -2129,14 +2247,14 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
let destination_lock = disk_namespace_mutation_lock(destination_object);
|
||||
let (namespace_guard, staging_guard) = match staging_prefix {
|
||||
Some(source) if source < destination_object => {
|
||||
let staging = disk_namespace_mutation_lock(source).lock_owned().await;
|
||||
(destination_lock.lock_owned().await, Some(staging))
|
||||
let staging = disk_namespace_mutation_lock(source).write_owned().await;
|
||||
(destination_lock.write_owned().await, Some(staging))
|
||||
}
|
||||
Some(source) if source > destination_object => {
|
||||
let destination = destination_lock.lock_owned().await;
|
||||
(destination, Some(disk_namespace_mutation_lock(source).lock_owned().await))
|
||||
let destination = destination_lock.write_owned().await;
|
||||
(destination, Some(disk_namespace_mutation_lock(source).write_owned().await))
|
||||
}
|
||||
_ => (destination_lock.lock_owned().await, None),
|
||||
_ => (destination_lock.write_owned().await, None),
|
||||
};
|
||||
let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
|
||||
let blocking_queue = disk_blocking_queue_for_root(root);
|
||||
@@ -2144,12 +2262,79 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
_namespace_guard: namespace_guard,
|
||||
_staging_guard: staging_guard,
|
||||
_volume_guard: Some(volume_guard),
|
||||
_secondary_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
blocking_queue,
|
||||
})
|
||||
}
|
||||
|
||||
/// Admit both physical volumes after the destination namespace, never taking
|
||||
/// the same READ twice when a bucket deletion is already queued.
|
||||
pub(in crate::disk) async fn acquire_rename_data_mutation_lease_for_volumes(
|
||||
root: &Path,
|
||||
source_volume: &str,
|
||||
destination_volume: &str,
|
||||
destination_object: &Path,
|
||||
staging_prefix: Option<&Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
let staging_guard = match staging_prefix {
|
||||
Some(source) if source < destination_object => Some(disk_namespace_mutation_lock(source).write_owned().await),
|
||||
_ => None,
|
||||
};
|
||||
let namespace = disk_namespace_mutation_lock(destination_object).write_owned();
|
||||
#[cfg(all(test, not(windows)))]
|
||||
let namespace_guard = {
|
||||
// Observe actual namespace contention, not a cooperative-budget yield.
|
||||
let mut namespace = std::pin::pin!(tokio::task::unconstrained(namespace));
|
||||
std::future::poll_fn(|cx| {
|
||||
let poll = std::future::Future::poll(namespace.as_mut(), cx);
|
||||
if poll.is_pending() {
|
||||
prepared_publication_test_hooks::run(
|
||||
prepared_publication_test_hooks::Stage::NamespaceWaiting,
|
||||
destination_object,
|
||||
);
|
||||
}
|
||||
poll
|
||||
})
|
||||
.await
|
||||
};
|
||||
#[cfg(not(all(test, not(windows))))]
|
||||
let namespace_guard = namespace.await;
|
||||
let staging_guard = match staging_prefix {
|
||||
Some(source) if source > destination_object => Some(disk_namespace_mutation_lock(source).write_owned().await),
|
||||
_ => staging_guard,
|
||||
};
|
||||
let source_key = root.join(source_volume);
|
||||
let destination_key = root.join(destination_volume);
|
||||
let (first, second) = if source_key <= destination_key {
|
||||
(source_volume, destination_volume)
|
||||
} else {
|
||||
(destination_volume, source_volume)
|
||||
};
|
||||
let first_guard = disk_volume_mutation_lock(root, first).read_owned().await;
|
||||
let second_guard = if source_key == destination_key {
|
||||
None
|
||||
} else {
|
||||
Some(disk_volume_mutation_lock(root, second).read_owned().await)
|
||||
};
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: namespace_guard,
|
||||
_staging_guard: staging_guard,
|
||||
_volume_guard: Some(first_guard),
|
||||
_secondary_volume_guard: second_guard,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
blocking_queue: disk_blocking_queue_for_root(root),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(windows)))]
|
||||
pub(crate) async fn drain_namespace_for_test(path: &Path) {
|
||||
drop(acquire_namespace_mutation_lease_with_owner(path, None).await);
|
||||
}
|
||||
|
||||
/// Always acquire the per-disk permit before the process-wide permit. Keeping
|
||||
/// this order uniform prevents one slow disk from reserving global capacity
|
||||
/// while it waits for its own concurrency slot.
|
||||
@@ -2382,6 +2567,8 @@ pub(crate) mod file_sync_probe {
|
||||
}
|
||||
|
||||
pub(crate) fn sync_file(path: &Path) -> io::Result<()> {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::FileSync, path);
|
||||
#[cfg(test)]
|
||||
let _probe = file_sync_probe::enter(path);
|
||||
#[cfg(test)]
|
||||
@@ -2536,6 +2723,15 @@ pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Keep the physical directory workers tied to their volume mutation owner.
|
||||
pub(crate) async fn make_dir_all_with_owner(path: &Path, base_dir: &Path, owner: Arc<dyn Send + Sync>) -> Result<()> {
|
||||
check_path_length(path.to_string_lossy().as_ref())?;
|
||||
reliable_mkdir_all_with_owner(path, base_dir, Some(owner))
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if a directory is empty.
|
||||
/// Only reads one entry to determine if the directory is empty.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
@@ -2700,6 +2896,8 @@ pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Ar
|
||||
queue
|
||||
.run_fsync_cancellable(move |_| {
|
||||
let _namespace_owner = namespace_owner;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &path);
|
||||
fsync_dir_std(path)
|
||||
})
|
||||
.await
|
||||
@@ -2731,6 +2929,10 @@ pub(crate) async fn remove_file_with_owner(
|
||||
.await;
|
||||
}
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
remove_file_with_lease(path, lease).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn remove_file_with_lease(path: PathBuf, lease: Arc<NamespaceMutationLease>) -> io::Result<()> {
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
@@ -2745,18 +2947,25 @@ pub(crate) async fn remove_files_and_fsync_with_owner(
|
||||
sync: bool,
|
||||
final_remove: Option<&str>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
prepared: Option<Arc<NamespaceMutationLease>>,
|
||||
) -> io::Result<()> {
|
||||
let mut leased_paths = Vec::with_capacity(names.len());
|
||||
// Follow caller order, keeping FINISHED before the final quorum receipt
|
||||
// to match rollback publication's overlapping namespace leases.
|
||||
for name in names {
|
||||
let path = dir.join(name);
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner.clone()).await;
|
||||
let lease = match &prepared {
|
||||
Some(publication) => publication.clone(),
|
||||
None => acquire_namespace_mutation_lease_with_owner(&path, namespace_owner.clone()).await,
|
||||
};
|
||||
leased_paths.push((path, lease));
|
||||
}
|
||||
let final_leased = if let Some(name) = final_remove {
|
||||
let path = dir.join(name);
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner.clone()).await;
|
||||
let lease = match &prepared {
|
||||
Some(publication) => publication.clone(),
|
||||
None => acquire_namespace_mutation_lease_with_owner(&path, namespace_owner.clone()).await,
|
||||
};
|
||||
Some((path, lease))
|
||||
} else {
|
||||
None
|
||||
@@ -2768,6 +2977,9 @@ pub(crate) async fn remove_files_and_fsync_with_owner(
|
||||
.map_or_else(|| disk_blocking_queue_for_path(dir), |(_, lease)| Arc::clone(&lease.blocking_queue));
|
||||
let operation = move |_: &dyn Fn() -> bool| {
|
||||
let _namespace_owner = namespace_owner;
|
||||
// A prepared transaction already owns these keys. Keep its complete
|
||||
// guard set through cleanup, fsync and final receipt removal.
|
||||
let _prepared = prepared;
|
||||
for (path, lease) in leased_paths {
|
||||
let _lease = lease;
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
@@ -2843,6 +3055,10 @@ pub(in crate::disk) async fn rename_with_namespace_owner(
|
||||
let src = src.to_path_buf();
|
||||
let dst = dst.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&dst, namespace_owner).await;
|
||||
rename_with_lease(src, dst, lease).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_with_lease(src: PathBuf, dst: PathBuf, lease: Arc<NamespaceMutationLease>) -> io::Result<()> {
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
{
|
||||
@@ -2863,6 +3079,10 @@ pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
create_dir_all_with_lease(path, lease).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn create_dir_all_with_lease(path: PathBuf, lease: Arc<NamespaceMutationLease>) -> io::Result<()> {
|
||||
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
|
||||
}
|
||||
|
||||
@@ -3188,9 +3408,13 @@ pub(crate) async fn rename_all_with_commit_guard(
|
||||
{
|
||||
let source = src_file_path.as_ref().to_path_buf();
|
||||
let destination = dst_file_path.as_ref().to_path_buf();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let mut physical_barrier = PhysicalRenameTestBarrier::selected(publication_root.path(), &destination, "waiter-dropped")?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let physical_barrier_path = physical_barrier.as_ref().map(|barrier| barrier.path.clone());
|
||||
let guard = commit_guard.clone();
|
||||
let _ = (base_dir, publication_root);
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let result = run_blocking_namespace_operation(lease, move || {
|
||||
guard.directories.prepare_destination()?;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run_rename_destination(&source, &destination);
|
||||
@@ -3199,11 +3423,20 @@ pub(crate) async fn rename_all_with_commit_guard(
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &source);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination);
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(path) = physical_barrier_path.as_ref() {
|
||||
PhysicalRenameTestBarrier::wait(path)?;
|
||||
}
|
||||
guard.directories.rename(&source, &destination, None)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
.map_err(DiskError::from);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(observer) = physical_barrier.as_mut() {
|
||||
observer.complete();
|
||||
}
|
||||
result
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
@@ -3235,8 +3468,18 @@ pub(crate) async fn rename_all_ignore_missing_source_with_owner(
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
rename_all_ignore_missing_source_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_all_ignore_missing_source_with_lease(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
match reliable_rename_inner_with_lease(
|
||||
src_file_path.to_path_buf(),
|
||||
dst_file_path.as_ref().to_path_buf(),
|
||||
@@ -3304,6 +3547,97 @@ async fn reliable_rename_inner(
|
||||
.await
|
||||
}
|
||||
|
||||
// The barrier runs inside the physical worker. Its async observer owns no lease,
|
||||
// and separately records cancellation so E2E tests cannot mistake an open RPC
|
||||
// waiter for ownership retained by a detached filesystem operation.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(in crate::disk) struct PhysicalRenameTestBarrier {
|
||||
path: PathBuf,
|
||||
completed: bool,
|
||||
cancellation_stage: &'static str,
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
impl PhysicalRenameTestBarrier {
|
||||
pub(in crate::disk) fn selected(
|
||||
root: &Path,
|
||||
destination: &Path,
|
||||
cancellation_stage: &'static str,
|
||||
) -> io::Result<Option<Self>> {
|
||||
let path = root.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
let expected = match std::fs::read_to_string(&path) {
|
||||
Ok(expected) => expected,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let Some(parent) = destination.parent() else { return Ok(None) };
|
||||
let Some(name) = destination.file_name() else { return Ok(None) };
|
||||
let Ok(parent) = parent.canonicalize() else { return Ok(None) };
|
||||
let destination = parent.join(name);
|
||||
// Construct lazily: dropping an unselected observer must never report
|
||||
// cancellation of another operation using this disk's barrier file.
|
||||
Ok((destination == Path::new(&expected)).then(|| Self {
|
||||
path,
|
||||
completed: false,
|
||||
cancellation_stage,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(in crate::disk) fn complete(&mut self) {
|
||||
self.completed = true;
|
||||
}
|
||||
|
||||
fn wait(path: &Path) -> io::Result<()> {
|
||||
// The synthetic pause retains physical ownership while unrelated disk work progresses.
|
||||
let _pause = enter_blocking_test_hook();
|
||||
std::fs::write(path.with_extension("arrived"), b"physical worker entered")?;
|
||||
let deadline = Instant::now() + Duration::from_secs(120);
|
||||
while !path.with_extension("release").try_exists()? {
|
||||
if Instant::now() >= deadline {
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "physical rename test barrier was not released"));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
impl Drop for PhysicalRenameTestBarrier {
|
||||
fn drop(&mut self) {
|
||||
if !self.completed {
|
||||
let _ = std::fs::write(self.path.with_extension(self.cancellation_stage), b"async waiter dropped");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "e2e-test-hooks"))]
|
||||
#[test]
|
||||
fn physical_rename_barrier_reports_only_selected_cancelled_waiters() {
|
||||
let root = tempfile::tempdir().expect("physical barrier fixture");
|
||||
let root_path = root.path().canonicalize().expect("canonical fixture root");
|
||||
let barrier = root_path.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
std::fs::create_dir_all(barrier.parent().expect("barrier parent")).expect("barrier directory");
|
||||
let destination = root_path.join("selected");
|
||||
std::fs::write(&barrier, destination.to_str().expect("fixture path")).expect("select physical destination");
|
||||
let marker = barrier.with_extension("waiter-dropped");
|
||||
assert!(
|
||||
PhysicalRenameTestBarrier::selected(&root_path, &root_path.join("other"), "waiter-dropped")
|
||||
.expect("unselected observer")
|
||||
.is_none()
|
||||
);
|
||||
assert!(!marker.exists(), "unselected rename falsely reported cancellation");
|
||||
drop(PhysicalRenameTestBarrier::selected(&root_path, &destination, "waiter-dropped").expect("cancelled observer"));
|
||||
assert!(marker.exists(), "selected cancelled waiter was not observed");
|
||||
std::fs::remove_file(&marker).expect("reset observation");
|
||||
let mut observer = PhysicalRenameTestBarrier::selected(&root_path, &destination, "waiter-dropped")
|
||||
.expect("completed observer")
|
||||
.expect("selected destination");
|
||||
observer.complete();
|
||||
drop(observer);
|
||||
assert!(!marker.exists(), "completed waiter falsely reported cancellation");
|
||||
}
|
||||
|
||||
async fn reliable_rename_inner_with_lease(
|
||||
src_file_path: PathBuf,
|
||||
dst_file_path: PathBuf,
|
||||
@@ -3312,6 +3646,10 @@ async fn reliable_rename_inner_with_lease(
|
||||
warn_on_missing_source: bool,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> io::Result<()> {
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let mut physical_barrier = PhysicalRenameTestBarrier::selected(publication_root.path(), &dst_file_path, "waiter-dropped")?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let physical_barrier_path = physical_barrier.as_ref().map(|barrier| barrier.path.clone());
|
||||
let operation = {
|
||||
let src_file_path = src_file_path.clone();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
@@ -3325,10 +3663,19 @@ async fn reliable_rename_inner_with_lease(
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(path) = physical_barrier_path.as_ref() {
|
||||
PhysicalRenameTestBarrier::wait(path)?;
|
||||
}
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
};
|
||||
let result = run_blocking_namespace_operation(lease, operation).await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(barrier) = physical_barrier.as_mut() {
|
||||
barrier.complete();
|
||||
}
|
||||
|
||||
if let Err(err) = &result
|
||||
&& (warn_on_missing_source || err.kind() != io::ErrorKind::NotFound)
|
||||
{
|
||||
@@ -5182,7 +5529,9 @@ pub(crate) fn compare_and_update_control_file(
|
||||
validate_windows_owned_file(&lock)?;
|
||||
match lock.as_file().try_lock() {
|
||||
Ok(()) => {}
|
||||
Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
|
||||
Err(std::fs::TryLockError::WouldBlock) => {
|
||||
return Err(io::Error::from(io::ErrorKind::WouldBlock));
|
||||
}
|
||||
Err(std::fs::TryLockError::Error(err)) => return Err(err),
|
||||
}
|
||||
|
||||
@@ -5612,11 +5961,15 @@ fn should_retry_prepare(err: &io::Error, attempt: usize, prune_budget: usize) ->
|
||||
}
|
||||
|
||||
pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
reliable_mkdir_all_with_owner(path.as_ref(), base_dir.as_ref(), None).await
|
||||
}
|
||||
|
||||
async fn reliable_mkdir_all_with_owner(path: &Path, base_dir: &Path, owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
let mut i = 0;
|
||||
|
||||
let mut base_dir = base_dir.as_ref();
|
||||
let mut base_dir = base_dir;
|
||||
loop {
|
||||
if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await {
|
||||
if let Err(e) = os_mkdir_all(path, base_dir, owner.clone()).await {
|
||||
if e.kind() == io::ErrorKind::NotFound && i == 0 {
|
||||
i += 1;
|
||||
|
||||
@@ -5638,16 +5991,34 @@ pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Pat
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mkdir_with_owner(path: &Path, recursive: bool, owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
let path = path.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = owner;
|
||||
#[cfg(test)]
|
||||
super::local::volume_mutation_test_hooks::run(super::local::volume_mutation_test_hooks::Stage::Create, &path);
|
||||
let result = if recursive {
|
||||
std::fs::create_dir_all(&path)
|
||||
} else {
|
||||
std::fs::create_dir(&path)
|
||||
};
|
||||
#[cfg(test)]
|
||||
super::local::volume_mutation_test_hooks::run(super::local::volume_mutation_test_hooks::Stage::CreateComplete, &path);
|
||||
result
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// Create a directory and all its parent components if they are missing.
|
||||
/// Without recursion support, fall back to create_dir_all
|
||||
/// This function will not create directories under base_dir.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) {
|
||||
async fn os_mkdir_all(dir_path: &Path, base_dir: &Path, owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
if !base_dir.to_string_lossy().is_empty() && base_dir.starts_with(dir_path) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Err(e) = super::fs::mkdir(dir_path.as_ref()).await {
|
||||
if let Err(e) = mkdir_with_owner(dir_path, false, owner.clone()).await {
|
||||
if e.kind() == io::ErrorKind::AlreadyExists {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -5656,16 +6027,16 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if let Some(parent) = dir_path.as_ref().parent() {
|
||||
if let Some(parent) = dir_path.parent() {
|
||||
// Fall back to creating the missing parent chain only when the direct mkdir proves it is required.
|
||||
if let Err(parent_err) = super::fs::make_dir_all(parent).await
|
||||
if let Err(parent_err) = mkdir_with_owner(parent, true, owner.clone()).await
|
||||
&& parent_err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(parent_err);
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(retry_err) = super::fs::mkdir(dir_path.as_ref()).await
|
||||
if let Err(retry_err) = mkdir_with_owner(dir_path, false, owner.clone()).await
|
||||
&& retry_err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(retry_err);
|
||||
@@ -5712,6 +6083,84 @@ mod tests {
|
||||
use tempfile::tempdir;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
#[tokio::test]
|
||||
async fn physical_rename_barrier_allows_control_work_without_releasing_cancelled_owner() {
|
||||
struct ReleaseBarrier(PathBuf);
|
||||
|
||||
impl Drop for ReleaseBarrier {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::write(self.0.with_extension("release"), b"release physical worker");
|
||||
}
|
||||
}
|
||||
|
||||
let temp = tempdir().expect("physical barrier fixture");
|
||||
let root = temp.path().canonicalize().expect("canonical disk root");
|
||||
let queue = register_disk_blocking_queue(&root, &root);
|
||||
let source = root.join("source");
|
||||
let destination = root.join("destination");
|
||||
std::fs::write(&source, b"physical payload").expect("staged payload");
|
||||
let metadata = root.join("control-metadata");
|
||||
std::fs::write(&metadata, b"control").expect("existing control metadata");
|
||||
let barrier = root.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
std::fs::create_dir_all(barrier.parent().expect("barrier parent")).expect("barrier directory");
|
||||
std::fs::write(&barrier, destination.to_str().expect("destination path")).expect("arm physical barrier");
|
||||
let release = ReleaseBarrier(barrier.clone());
|
||||
let lease = acquire_namespace_mutation_lease(&destination).await;
|
||||
assert!(
|
||||
Arc::ptr_eq(&lease.blocking_queue, &queue),
|
||||
"control and physical work share the disk queue"
|
||||
);
|
||||
let owner = Arc::downgrade(&lease);
|
||||
let publication_root = PublicationRoot::new(&root).expect("pinned disk root");
|
||||
let worker_destination = destination.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
reliable_rename_inner_with_lease(source, worker_destination, root, publication_root, false, lease).await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !barrier.with_extension("arrived").exists() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical worker enters the real E2E barrier");
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("cancel async rename waiter").is_cancelled());
|
||||
assert!(barrier.with_extension("waiter-dropped").exists(), "receiver observes waiter cancellation");
|
||||
|
||||
let control_owner = owner.clone();
|
||||
let control_destination = destination.clone();
|
||||
let control = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
queue.run(move || {
|
||||
Ok((
|
||||
std::fs::read(metadata)?,
|
||||
control_owner.upgrade().is_some(),
|
||||
disk_namespace_mutation_lock(&control_destination).try_write_owned().is_err(),
|
||||
control_destination.exists(),
|
||||
))
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
// Drain even when the progress oracle fails, so a regression does not strand a worker.
|
||||
drop(release);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while owner.upgrade().is_some() || !destination.exists() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical completion releases the namespace owner");
|
||||
let (bytes, retained, locked, published) = control
|
||||
.expect("control work progresses during the pause")
|
||||
.expect("control read");
|
||||
assert_eq!(bytes, b"control");
|
||||
assert!(retained && locked, "cancelled waiter must leave the physical namespace lease owned");
|
||||
assert!(!published, "control work must run before physical publication");
|
||||
assert!(disk_namespace_mutation_lock(&destination).try_write_owned().is_ok());
|
||||
assert_eq!(std::fs::read(destination).expect("completed physical publication"), b"physical payload");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_staging_leases_are_ordered_and_deduplicated() {
|
||||
let root = tempdir().expect("namespace fixture");
|
||||
@@ -5727,7 +6176,7 @@ mod tests {
|
||||
);
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while disk_namespace_mutation_lock(&early).try_lock_owned().is_ok() {
|
||||
while disk_namespace_mutation_lock(&early).try_write_owned().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
@@ -9370,6 +9819,7 @@ mod tests {
|
||||
sync,
|
||||
Some("receipt"),
|
||||
None,
|
||||
None,
|
||||
));
|
||||
assert!(futures::FutureExt::now_or_never(&mut cleanup).is_none());
|
||||
let foreground = tokio::time::timeout(Duration::from_secs(5), queue.run(|| Ok(()))).await;
|
||||
@@ -9384,6 +9834,56 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn prepared_cleanup_reuses_lease_and_retains_it_after_cancellation() {
|
||||
let root = tempdir().expect("disk root");
|
||||
let intermediate = root.path().join("intermediate");
|
||||
let receipt = root.path().join("receipt");
|
||||
std::fs::write(&intermediate, b"backup").expect("backup");
|
||||
std::fs::write(&receipt, b"committed").expect("receipt");
|
||||
let prepared = acquire_namespace_mutation_lease(&intermediate).await;
|
||||
let probe = Arc::downgrade(&prepared);
|
||||
let (entered_tx, entered_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = prepared_publication_test_hooks::install_at(
|
||||
prepared_publication_test_hooks::Stage::Remove,
|
||||
&intermediate,
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let dir = root.path().to_path_buf();
|
||||
let worker = tokio::spawn(async move {
|
||||
remove_files_and_fsync_with_owner(&dir, &["intermediate"], true, Some("receipt"), None, Some(prepared)).await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("prepared cleanup must reuse its existing key instead of waiting for itself")
|
||||
.expect("physical cleanup entered");
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("caller cancelled").is_cancelled());
|
||||
let retained = probe.upgrade().is_some();
|
||||
let locked = disk_namespace_mutation_lock(&intermediate).try_write_owned().is_err();
|
||||
release_tx.send(()).expect("release physical cleanup");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while probe.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical cleanup must release its owner");
|
||||
assert!(
|
||||
retained && locked,
|
||||
"prepared cleanup must keep its complete guard set after caller cancellation"
|
||||
);
|
||||
assert!(
|
||||
!intermediate.exists() && !receipt.exists(),
|
||||
"cleanup must remove intermediates before retiring the receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_blocking_queue_rejected_worker_releases_jobs_and_resources() {
|
||||
use crate::disk::io_admission::with_background_resource_unit;
|
||||
@@ -9733,15 +10233,16 @@ mod tests {
|
||||
let file_c = temp_dir.path().join("c");
|
||||
std::fs::write(&file_a, b"a").expect("write a");
|
||||
std::fs::write(&file_c, b"c").expect("write c");
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["a", "missing-b"], true, Some("c"), None)
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["a", "missing-b"], true, Some("c"), None, None)
|
||||
.await
|
||||
.expect("remove_files_and_fsync_with_owner must tolerate missing intermediates and remove final");
|
||||
assert!(!file_a.exists() && !file_c.exists());
|
||||
|
||||
// Missing final_remove must propagate NotFound.
|
||||
let missing_final = remove_files_and_fsync_with_owner(temp_dir.path(), &["missing-a"], false, Some("missing-c"), None)
|
||||
.await
|
||||
.expect_err("missing final file must return an error");
|
||||
let missing_final =
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["missing-a"], false, Some("missing-c"), None, None)
|
||||
.await
|
||||
.expect_err("missing final file must return an error");
|
||||
assert_eq!(missing_final.kind(), io::ErrorKind::NotFound);
|
||||
|
||||
// Non-NotFound failure on an intermediate entry must abort before final_remove.
|
||||
@@ -9750,7 +10251,7 @@ mod tests {
|
||||
std::fs::create_dir(&blocker_dir).expect("create directory blocker");
|
||||
std::fs::write(&preserved_final, b"keep").expect("write preserved final");
|
||||
assert!(
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["blocker"], true, Some("preserved-final"), None)
|
||||
remove_files_and_fsync_with_owner(temp_dir.path(), &["blocker"], true, Some("preserved-final"), None, None)
|
||||
.await
|
||||
.is_err(),
|
||||
"directory in place of file must fail intermediate remove_file"
|
||||
|
||||
@@ -153,10 +153,13 @@ async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters()
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep
|
||||
/// them alive for the test's duration and the directories are removed on drop.
|
||||
pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
|
||||
make_local_set_disks_with_ctx(drive_count, parity_count, crate::runtime::instance::bootstrap_ctx()).await
|
||||
/// Keep the real metadata owner and backing directories alive together.
|
||||
pub(crate) async fn make_local_set_disks(
|
||||
drive_count: usize,
|
||||
parity_count: usize,
|
||||
) -> (crate::set_disk::HermeticSetFixture, Arc<SetDisks>) {
|
||||
let (fixture, _, set) = crate::set_disk::hermetic_set_disks_for_pool_with_default_parity(drive_count, 0, parity_count).await;
|
||||
(fixture, set)
|
||||
}
|
||||
|
||||
/// Like [`make_local_set_disks`], but binds the set to an explicit instance
|
||||
@@ -756,6 +759,9 @@ async fn blackbox_range_read_restores_exact_slice_with_one_offline_disk() {
|
||||
#[tokio::test]
|
||||
async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_metadata() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = crate::bucket::metadata_sys::object_store_in(set_disks.instance_ctx())
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bb-delete-marker-read-negative";
|
||||
let object = "object.bin";
|
||||
let opts = ObjectOptions {
|
||||
@@ -765,13 +771,26 @@ async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"body hidden by delete marker".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("object should be written before delete marker");
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ mod storage_api_contracts;
|
||||
mod store;
|
||||
|
||||
pub use disk::{with_background_disk_io, with_background_resource_unit};
|
||||
pub use store::PoolMetaWriteGateStatus;
|
||||
pub use store::{BucketIncarnationFenceGuard, PoolMetaWriteGateStatus};
|
||||
|
||||
// pub mod checksum;
|
||||
mod event;
|
||||
|
||||
@@ -4473,3 +4473,7 @@ mod tests {
|
||||
.expect_err("a lying frame-layout marker must fail closed, not serve plaintext");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "readers/compression_index_compatibility_tests.rs"]
|
||||
mod compression_index_compatibility_tests;
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
use super::*;
|
||||
use rustfs_utils::http::metadata_compat::{SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, insert_str};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const STREAM: &[u8] = include_bytes!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/compression-index/legacy-lz4-stream.bin"
|
||||
));
|
||||
const HISTORICAL_INDEX: &[u8] = include_bytes!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/compression-index/pre-zigzag-index.bin"
|
||||
));
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
const CURRENT_INDEX: &[u8] =
|
||||
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/compression-index/zigzag-index.bin"));
|
||||
const PLAINTEXT_LEN: usize = 3 * 1024 * 1024 + 123;
|
||||
const PLAINTEXT_SHA256: &str = "68f9edf6f9600395d6d2b5517d97570fecd2a0c6dbe6130617ef415a3b48d89c";
|
||||
|
||||
fn fixture_info(index: &[u8]) -> ObjectInfo {
|
||||
let mut metadata = HashMap::new();
|
||||
insert_str(&mut metadata, SUFFIX_COMPRESSION, "lz4".to_owned());
|
||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, PLAINTEXT_LEN.to_string());
|
||||
ObjectInfo {
|
||||
size: i64::try_from(STREAM.len()).expect("fixed fixture size fits i64"),
|
||||
parts: Arc::new(vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: STREAM.len(),
|
||||
actual_size: i64::try_from(PLAINTEXT_LEN).expect("fixed plaintext length fits i64"),
|
||||
index: Some(Bytes::copy_from_slice(index)),
|
||||
..Default::default()
|
||||
}]),
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture_plaintext() -> Vec<u8> {
|
||||
(0..PLAINTEXT_LEN)
|
||||
.map(|offset| {
|
||||
let chunk = offset / 1024;
|
||||
let lane = offset % 1024;
|
||||
if lane < 8 {
|
||||
u64::try_from(chunk).expect("fixed chunk number fits u64").to_le_bytes()[lane]
|
||||
} else {
|
||||
b"abcdefghijklmnopqrstuvwxyz"[(offset + chunk) % 26]
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn historical_index_preserves_the_complete_compressed_body() {
|
||||
let info = fixture_info(HISTORICAL_INDEX);
|
||||
let (mut reader, storage_offset, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(STREAM.to_vec())),
|
||||
None,
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct reader for fixed historical index");
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.expect("read complete historical body");
|
||||
assert_eq!(storage_offset, 0);
|
||||
assert_eq!(actual.len(), PLAINTEXT_LEN);
|
||||
assert_eq!(
|
||||
hex_simd::encode_to_string(Sha256::digest(&actual), hex_simd::AsciiCase::Lower),
|
||||
PLAINTEXT_SHA256
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn historical_index_preserves_ranges_across_compression_blocks() {
|
||||
let info = fixture_info(HISTORICAL_INDEX);
|
||||
let expected = fixture_plaintext();
|
||||
for start in [7, 1024 * 1024 - 17, 2 * 1024 * 1024 + 31, PLAINTEXT_LEN - 97] {
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: i64::try_from(start).expect("fixed range start fits i64"),
|
||||
end: i64::try_from(start + 63).expect("fixed range end fits i64"),
|
||||
};
|
||||
let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.await
|
||||
.expect("plan historical compressed range");
|
||||
let physical = plan.storage_offset;
|
||||
let input = STREAM.get(physical..).expect("planned offset stays within the fixed stream");
|
||||
let (mut reader, _, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(input.to_vec())),
|
||||
Some(range),
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct historical range reader");
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.expect("read historical range");
|
||||
assert_eq!(actual, expected[start..start + 64], "range starting at {start}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
#[tokio::test]
|
||||
async fn current_index_selects_a_fixed_compressed_block_for_a_range() {
|
||||
let info = fixture_info(CURRENT_INDEX);
|
||||
let start = 2 * 1024 * 1024 + 31;
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: i64::try_from(start).expect("fixed range start fits i64"),
|
||||
end: i64::try_from(start + 63).expect("fixed range end fits i64"),
|
||||
};
|
||||
let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.await
|
||||
.expect("plan range with fixed current index");
|
||||
assert_eq!(plan.storage_offset, 34_663, "use the recorded second block boundary");
|
||||
let physical = plan.storage_offset;
|
||||
let (mut reader, _, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(STREAM[physical..].to_vec())),
|
||||
Some(range),
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct indexed range reader");
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.expect("read indexed range");
|
||||
assert_eq!(actual, fixture_plaintext()[start..start + 64]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn truncated_index_falls_back_without_truncating_the_range() {
|
||||
let info = fixture_info(&HISTORICAL_INDEX[..HISTORICAL_INDEX.len() - 3]);
|
||||
let start = 2 * 1024 * 1024 + 31;
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: i64::try_from(start).expect("fixed range start fits i64"),
|
||||
end: i64::try_from(start + 96).expect("fixed range end fits i64"),
|
||||
};
|
||||
let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.await
|
||||
.expect("invalid index still permits a sequential range plan");
|
||||
assert_eq!(plan.storage_offset, 0, "truncated index must not select a storage offset");
|
||||
let (mut reader, _, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(STREAM.to_vec())),
|
||||
Some(range),
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct sequential fallback reader");
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read range without the truncated index");
|
||||
assert_eq!(actual, fixture_plaintext()[start..start + 97]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn historical_index_fallback_does_not_hide_a_body_checksum_failure() {
|
||||
let info = fixture_info(HISTORICAL_INDEX);
|
||||
let mut corrupted = STREAM.to_vec();
|
||||
corrupted[4] ^= 1; // Corrupt the first block's stored CRC, leaving its payload intact.
|
||||
let (mut reader, _, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(corrupted)),
|
||||
None,
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct reader before body checksum verification");
|
||||
let mut actual = Vec::new();
|
||||
let error = reader
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect_err("bad body CRC cannot become successful EOF");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(actual.len() < PLAINTEXT_LEN);
|
||||
}
|
||||
@@ -152,7 +152,7 @@ pub struct ObjectLockConfigSnapshot {
|
||||
config_revision: Option<OffsetDateTime>,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
_lifecycle_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
}
|
||||
|
||||
@@ -210,11 +210,36 @@ impl ObjectLockConfigSnapshot {
|
||||
config_revision: Some(config_revision),
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
_lifecycle_guard: Some(Arc::new(lifecycle_guard)),
|
||||
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_store_bucket_under_lifecycle_owner(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
config_revision: OffsetDateTime,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Self {
|
||||
let mut lifecycle_fence = NamespaceLockFence::new();
|
||||
if let Some(signal) = lifecycle_guard.lock_lost_signal() {
|
||||
lifecycle_fence.add_signal(signal);
|
||||
}
|
||||
Self {
|
||||
store_id: Some(store_id),
|
||||
bucket: Some(bucket.to_owned()),
|
||||
bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
config_revision: Some(config_revision),
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
metadata_transaction_guard: Some(metadata_transaction_guard),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_store_bucket_under_lifecycle_fence(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
@@ -254,6 +279,15 @@ impl ObjectLockConfigSnapshot {
|
||||
&self.state
|
||||
}
|
||||
|
||||
pub(crate) fn is_unbound(&self) -> bool {
|
||||
self.store_id.is_none()
|
||||
&& self.bucket.is_none()
|
||||
&& self.bucket_incarnation_id.is_none()
|
||||
&& self.config_revision.is_none()
|
||||
&& self._lifecycle_guard.is_none()
|
||||
&& self.metadata_transaction_guard.is_none()
|
||||
}
|
||||
|
||||
pub(crate) fn is_valid_for_destructive_put(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
|
||||
self.store_id == Some(store_id)
|
||||
&& self.bucket.as_deref() == Some(bucket)
|
||||
@@ -266,6 +300,11 @@ impl ObjectLockConfigSnapshot {
|
||||
.is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
pub(crate) fn owns_bucket_lifecycle_fence(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
|
||||
self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
|
||||
&& self._lifecycle_guard.as_ref().is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
/// Share the held transaction lock without queuing another reader behind
|
||||
/// a metadata writer that is itself waiting for this snapshot to drop.
|
||||
pub(crate) fn metadata_transaction_guard_for(
|
||||
@@ -962,6 +1001,12 @@ pub struct ObjectOptions {
|
||||
/// A genuine target write guard; loss fences alone do not authorize a borrowed write.
|
||||
#[doc(hidden)]
|
||||
pub write_lock_context: Option<WriteLockContext>,
|
||||
/// Ordinary ECStore PUT target bound to its bucket's immutable artifact namespace.
|
||||
#[doc(hidden)]
|
||||
pub artifact_put_target: Option<Arc<crate::bucket::artifact::ArtifactPutTarget>>,
|
||||
/// Owned, non-serializable authority for one fully sealed managed version.
|
||||
#[doc(hidden)]
|
||||
pub artifact_delete_target: Option<Arc<crate::bucket::artifact::ArtifactDeleteTarget>>,
|
||||
/// In-process delete ownership for late filesystem completion. Never
|
||||
/// serialized or copied into object metadata.
|
||||
#[doc(hidden)]
|
||||
@@ -1041,6 +1086,9 @@ pub struct ObjectOptions {
|
||||
/// Proves an upper layer holds the bucket lifecycle sentinel. A separate
|
||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||
/// Actual generation owner retained by storage commits and their physical tails.
|
||||
#[doc(hidden)]
|
||||
pub bucket_incarnation_fence: Option<Arc<crate::store::BucketIncarnationFenceGuard>>,
|
||||
/// Retains the actual lifecycle guard for an owned multipart commit. A
|
||||
/// lock-loss fence alone does not keep a cancelled caller's lock alive. The
|
||||
/// owner also retains the continuous incarnation-validation coverage.
|
||||
|
||||
@@ -71,7 +71,7 @@ impl WriteCommitGuard {
|
||||
|
||||
/// Namespace locks key the disk identity of the name (see `new_ns_lock`),
|
||||
/// so a guard proves every spelling that resolves to the same file.
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
pub(crate) fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
self.bucket.as_ref() == bucket && self.object.as_ref() == object_name_disk_identity(object).as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -814,6 +814,14 @@ impl InstanceContext {
|
||||
self.tier_config_mgr.get_or_init(TierConfigMgr::new).clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_test_tier_config_mgr(&self, manager: Arc<RwLock<TierConfigMgr>>) {
|
||||
assert!(
|
||||
self.tier_config_mgr.set(manager).is_ok(),
|
||||
"test tier manager must be installed before use"
|
||||
);
|
||||
}
|
||||
|
||||
/// This instance's event notifier, materializing it on first access.
|
||||
/// Shared for the lifetime of the instance.
|
||||
pub fn event_notifier(&self) -> Arc<RwLock<EventNotifier>> {
|
||||
|
||||
@@ -250,7 +250,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let bucket_incarnation_fence = match bucket_configs.bucket_incarnation_id {
|
||||
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||
Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
|
||||
None => None,
|
||||
};
|
||||
|
||||
@@ -533,9 +533,8 @@ impl ECStore {
|
||||
&cleanup_transferred_tier_versions,
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: bucket_configs.bucket_incarnation_id,
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
lifecycle_owner: bucket_incarnation_fence.as_ref(),
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: lock_lost_signal.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -1729,6 +1728,8 @@ mod tests {
|
||||
/// Both versions are on the target with their identities, the null
|
||||
/// version's content and SHA-256 are intact, and the source is gone.
|
||||
async fn assert_moved(&self) {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let target = self.store.pools[1]
|
||||
.get_disks_by_key(self.object)
|
||||
.load_file_info_versions_exact(self.bucket, self.object)
|
||||
@@ -1769,6 +1770,16 @@ mod tests {
|
||||
self.source_versions().await.is_none(),
|
||||
"the source entry must be removed only after both versions moved"
|
||||
);
|
||||
let source_disks = self.source_set.disk_inventory().await;
|
||||
assert_eq!(source_disks.iter().flatten().count(), 2, "both source disks must be online");
|
||||
for disk in source_disks.into_iter().flatten() {
|
||||
assert!(
|
||||
!tokio::fs::try_exists(disk.path().join(self.bucket).join(self.object))
|
||||
.await
|
||||
.expect("inspect the physical rebalance source path"),
|
||||
"rebalance must remove source metadata and shards from every disk"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -88,6 +88,7 @@ pub async fn test_store_with_persisted_rebalance_meta(
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
(temp_dirs, store)
|
||||
}
|
||||
@@ -266,6 +267,7 @@ async fn test_pool_stores_with_contexts(
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
|
||||
ctx: store_ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
};
|
||||
let store = make_store(ctx);
|
||||
|
||||
@@ -3415,6 +3415,7 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -3053,7 +3053,7 @@ impl RenameRollbackReceipt {
|
||||
|
||||
struct RenameRollbackOwnership {
|
||||
receipt: Option<RenameRollbackReceipt>,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
async fn inspect_incomplete_rename_rollback(
|
||||
@@ -3169,7 +3169,7 @@ async fn rollback_failed_rename(
|
||||
old_data_dir: rollback_dir,
|
||||
..Default::default()
|
||||
},
|
||||
disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
disk_namespace_commit_guard,
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -3281,10 +3281,11 @@ async fn record_rename_rollback_outcomes(
|
||||
/// quorum and optional scanner lease map together avoids widening either
|
||||
/// fanout helper's argument list while preserving the fence semantics.
|
||||
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
write_quorum: usize,
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
}
|
||||
|
||||
@@ -3294,6 +3295,7 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
artifact_put: None,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope: None,
|
||||
@@ -3302,6 +3304,14 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_artifact_put(
|
||||
mut self,
|
||||
identity: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
) -> Self {
|
||||
self.artifact_put = identity;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self {
|
||||
self.rollback_receipt = Some(receipt);
|
||||
self
|
||||
@@ -3317,7 +3327,7 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
|
||||
pub(in crate::set_disk) fn with_namespace_commit_guard(
|
||||
mut self,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Self {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
@@ -3770,6 +3780,7 @@ impl SetDisks {
|
||||
fence_options: RenameDataFenceOptions<'_>,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
let RenameDataFenceOptions {
|
||||
artifact_put,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope: _scanner_publication_commit_scope,
|
||||
@@ -3864,11 +3875,10 @@ impl SetDisks {
|
||||
&file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
crate::disk::RenameDataGuards {
|
||||
artifact_put,
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
namespace_owner: namespace_commit_guard.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4177,6 +4187,7 @@ impl SetDisks {
|
||||
.await;
|
||||
}
|
||||
let RenameDataFenceOptions {
|
||||
artifact_put,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope,
|
||||
@@ -4297,11 +4308,10 @@ impl SetDisks {
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
crate::disk::RenameDataGuards {
|
||||
artifact_put,
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
namespace_owner: namespace_commit_guard.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -12815,6 +12825,118 @@ mod tests {
|
||||
assert!(missing.below_quorum, "missing disk slots should remain visible in the quorum lens");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn legacy_path_delete_flow_prefix_delete_keeps_scanner_owner_until_drain() {
|
||||
use crate::disk::os::{drain_namespace_for_test, prepared_publication_test_hooks as hooks};
|
||||
use crate::object_api::{
|
||||
ScannerPublicationCommitScope, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
|
||||
};
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::storage_api_contracts::bucket::BucketOperations;
|
||||
|
||||
let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "path-delete-scanner";
|
||||
crate::bucket::metadata_sys::object_store_in(&set.ctx)
|
||||
.await
|
||||
.expect("fixture object store")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("authoritative bucket creation");
|
||||
let mut paths = Vec::new();
|
||||
for disk in &disks {
|
||||
disk.write_all(bucket, "prefix/part.1", Bytes::from_static(b"old shard"))
|
||||
.await
|
||||
.expect("seed prefix on every disk");
|
||||
paths.push(
|
||||
disk.get_object_path_for_io_if_local(bucket, "prefix")
|
||||
.expect("local disk")
|
||||
.expect("prefix path"),
|
||||
);
|
||||
}
|
||||
let movement = Arc::new(tokio::sync::RwLock::new(()));
|
||||
let scope = ScannerPublicationCommitScope::new_storage_owned(
|
||||
7,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
Vec::new(),
|
||||
movement.clone().read_owned().await,
|
||||
);
|
||||
scope.try_begin().expect("begin scanner publication");
|
||||
let caller_guard = ScannerPublicationCommitScopeGuard::new(scope.clone());
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install_at(hooks::Stage::Rename, &paths[0], move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let owner_set = set.clone();
|
||||
let owner_scope = scope.clone();
|
||||
let caller = tokio::spawn(async move {
|
||||
owner_set
|
||||
.delete_prefix_with_scanner_publication_lease(bucket, "prefix", None, Some(owner_scope))
|
||||
.await
|
||||
});
|
||||
let entered = tokio::time::timeout(Duration::from_secs(5), entered_rx).await;
|
||||
caller.abort();
|
||||
let _ = caller.await;
|
||||
drop(caller_guard);
|
||||
let state_while_blocked = scope.state();
|
||||
let movement_while_blocked = movement.clone().try_write_owned().is_ok();
|
||||
let disk::Disk::Local(local) = disks[0].as_ref() else {
|
||||
unreachable!("fixture contains local disks");
|
||||
};
|
||||
let local = local.get_disk();
|
||||
let volume_read_held = crate::disk::os::disk_volume_mutation_lock(&local.root, bucket)
|
||||
.try_write_owned()
|
||||
.is_err();
|
||||
let mut deletion = Box::pin(local.delete_volume(bucket, true));
|
||||
let early_deletion = futures::poll!(deletion.as_mut());
|
||||
let deletion_was_blocked = early_deletion.is_pending();
|
||||
// Always release and drain physical work before reporting an ownership failure.
|
||||
let _ = release_tx.send(());
|
||||
let mut drained = Vec::new();
|
||||
for path in &paths {
|
||||
drained.push(tokio::time::timeout(Duration::from_secs(5), drain_namespace_for_test(path)).await);
|
||||
}
|
||||
let deletion_result = match early_deletion {
|
||||
std::task::Poll::Ready(result) => result,
|
||||
std::task::Poll::Pending => tokio::time::timeout(Duration::from_secs(5), deletion)
|
||||
.await
|
||||
.expect("volume deletion drains"),
|
||||
};
|
||||
let terminal = tokio::time::timeout(Duration::from_secs(5), scope.wait_for_completion()).await;
|
||||
assert!(
|
||||
entered.is_ok_and(|result| result.is_ok()),
|
||||
"real prefix deletion must reach its filesystem worker"
|
||||
);
|
||||
assert!(drained.into_iter().all(|result| result.is_ok()), "every physical namespace must drain");
|
||||
deletion_result.expect("volume deletion after physical drain");
|
||||
assert!(
|
||||
volume_read_held && deletion_was_blocked,
|
||||
"volume deletion must wait for the physical prefix worker"
|
||||
);
|
||||
assert_eq!(state_while_blocked, ScannerPublicationCommitState::InFlight);
|
||||
assert!(!movement_while_blocked, "scanner movement remains fenced after caller cancellation");
|
||||
assert_eq!(terminal.expect("scanner owner completion"), ScannerPublicationCommitState::Committed);
|
||||
assert!(scope.release_movement_permit().await, "terminal scanner owner releases movement");
|
||||
assert!(movement.try_write_owned().is_ok());
|
||||
for path in paths.iter().skip(1) {
|
||||
assert!(!path.exists(), "successful prefix fanout removes every surviving volume's prefix");
|
||||
}
|
||||
disks[0].make_volume(bucket).await.expect("recreate deleted volume");
|
||||
disks[0]
|
||||
.write_all(bucket, "prefix/sentinel", Bytes::from_static(b"new generation"))
|
||||
.await
|
||||
.expect("new generation sentinel");
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.read_all(bucket, "prefix/sentinel")
|
||||
.await
|
||||
.expect("sentinel survives old deletion"),
|
||||
b"new generation".as_slice()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_succeeds_when_present_disks_reach_quorum() {
|
||||
let bucket = "delete-prefix-bucket";
|
||||
|
||||
@@ -906,6 +906,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::hermetic_set_disks_support::{HermeticSetFixture, hermetic_set_disks_for_pool_with_default_parity};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
@@ -1145,6 +1147,310 @@ mod prepared_get_object_metadata_tests {
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const OBSERVATION_HOOK_BODY_LEN: usize = 1024 * 1024 + 1;
|
||||
|
||||
struct ObservationBodyHook {
|
||||
object: &'static str,
|
||||
calls: Arc<AtomicU64>,
|
||||
body: bytes::Bytes,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::object_api::GetObjectBodyCacheHook for ObservationBodyHook {
|
||||
async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<bytes::Bytes> {
|
||||
if bucket == crate::disk::RUSTFS_META_BUCKET && object == self.object {
|
||||
assert_eq!(info.size, i64::try_from(self.body.len()).expect("hook fixture size"));
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
Some(self.body.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationHookGuard;
|
||||
|
||||
impl Drop for ObservationHookGuard {
|
||||
fn drop(&mut self) {
|
||||
crate::object_api::unregister_get_object_body_cache_hook();
|
||||
}
|
||||
}
|
||||
|
||||
async fn ordinary_observation_get(set: &SetDisks, object: &str) -> Vec<u8> {
|
||||
let mut reader = set
|
||||
.get_object_reader(crate::disk::RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("ordinary GET positive control");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("ordinary GET body");
|
||||
bytes
|
||||
}
|
||||
|
||||
fn observation_register_hook(object: &'static str, body: Vec<u8>) -> (ObservationHookGuard, Arc<AtomicU64>) {
|
||||
let calls = Arc::new(AtomicU64::new(0));
|
||||
crate::object_api::register_get_object_body_cache_hook(Arc::new(ObservationBodyHook {
|
||||
object,
|
||||
calls: calls.clone(),
|
||||
body: bytes::Bytes::from(body),
|
||||
}));
|
||||
(ObservationHookGuard, calls)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn locked_observation_uses_real_outer_guard() {
|
||||
let object = "config/locked-observation-real.bin";
|
||||
let payload = vec![b'p'; OBSERVATION_HOOK_BODY_LEN];
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist locked observation fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; payload.len()]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let mut contender = Box::pin(crate::object_api::WriteCommitGuard::acquire(
|
||||
&namespace,
|
||||
std::time::Duration::from_secs(30),
|
||||
));
|
||||
assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
crate::config::com::read_config_observation_no_lock(set.clone(), object, payload.len(), &guard),
|
||||
)
|
||||
.await
|
||||
.expect("locked observation hang watchdog");
|
||||
assert!(result.is_ok(), "I06L_NO_NESTED_LOCK: {result:?}");
|
||||
let (bytes, info) = result.expect("locked observation");
|
||||
assert!(bytes == payload, "I06L_HOOK_BYPASS");
|
||||
assert!(calls.load(Ordering::SeqCst) == 0, "I06L_HOOK_BYPASS");
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.data_dir, written.data_dir);
|
||||
assert_eq!(info.name, object);
|
||||
assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
|
||||
drop(guard);
|
||||
let next = tokio::time::timeout(std::time::Duration::from_secs(10), contender)
|
||||
.await
|
||||
.expect("owner release watchdog")
|
||||
.expect("competing owner acquired");
|
||||
drop(next);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn locked_observation_preserves_empty_missing_and_bound() {
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
for (object, body) in [
|
||||
("config/locked-empty.bin", Vec::new()),
|
||||
("config/locked-bound.bin", b"raw".to_vec()),
|
||||
] {
|
||||
let mut reader = PutObjReader::from_vec(body.clone());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist boundary fixture");
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching boundary owner");
|
||||
let result = crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len(), &guard).await;
|
||||
assert!(result.is_ok(), "I06L_EMPTY_DISTINCT: {result:?}");
|
||||
let (bytes, info) = result.expect("boundary observation");
|
||||
assert_eq!(bytes, body);
|
||||
assert_eq!(info.size, written.size);
|
||||
if !body.is_empty() {
|
||||
let result =
|
||||
crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len() - 1, &guard).await;
|
||||
assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_BOUND_ENFORCED");
|
||||
}
|
||||
}
|
||||
let object = "config/locked-absent.bin";
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching missing owner");
|
||||
let result = crate::config::com::read_config_observation_no_lock(set, object, 0, &guard).await;
|
||||
assert!(matches!(result, Err(Error::ConfigNotFound)), "I06L_EMPTY_DISTINCT");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_real_backend_boundaries() {
|
||||
let object = "config/observation-real.bin";
|
||||
let payload = b"persisted config";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(payload.to_vec());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let result = crate::config::com::read_config_observation(set.clone(), object, payload.len()).await;
|
||||
assert!(result.is_ok(), "I05_OBSERVATION_SUCCESS: {result:?}");
|
||||
let (bytes, info) = result.expect("real observation success");
|
||||
assert_eq!(bytes, payload);
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.data_dir, written.data_dir);
|
||||
assert_eq!(info.name, object);
|
||||
let over = crate::config::com::read_config_observation(set.clone(), object, payload.len() - 1).await;
|
||||
assert!(over.is_err(), "I05_BOUND_ENFORCED");
|
||||
assert_eq!(over.expect_err("over limit must fail"), Error::CorruptedFormat, "I05_BOUND_ENFORCED");
|
||||
let absent = crate::config::com::read_config_observation(set.clone(), "config/observation-absent.bin", 32).await;
|
||||
assert_eq!(absent.expect_err("absent config"), Error::ConfigNotFound);
|
||||
let empty_object = "config/observation-empty.bin";
|
||||
let mut empty = PutObjReader::from_vec(Vec::new());
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, empty_object, &mut empty, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist empty config");
|
||||
let empty = crate::config::com::read_config_observation(set, empty_object, 0).await;
|
||||
assert!(empty.is_ok(), "existing empty config must succeed: {empty:?}");
|
||||
let (bytes, info) = empty.expect("empty observation");
|
||||
assert!(bytes.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_bypasses_registered_body_hook() {
|
||||
let object = "config/observation-hook.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let result = crate::config::com::read_config_observation(set, object, OBSERVATION_HOOK_BODY_LEN).await;
|
||||
assert!(result.is_ok(), "I05_HOOK_BYPASS: {result:?}");
|
||||
assert_eq!(
|
||||
result.expect("observed persisted body").0,
|
||||
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
|
||||
"I05_HOOK_BYPASS"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0, "I05_HOOK_BYPASS");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_hook_scope_restores_after_error() {
|
||||
let object = "config/observation-hook-error.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
let result = crate::config::com::read_config_observation(set.clone(), object, OBSERVATION_HOOK_BODY_LEN - 1).await;
|
||||
assert_eq!(result.expect_err("bounded observation error"), Error::CorruptedFormat);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
struct PendingObservationStore {
|
||||
set: Arc<SetDisks>,
|
||||
ready: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PendingObservationStore {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("PendingObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for PendingObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
let mut reader = self.set.get_object_reader(bucket, object, range, headers, opts).await?;
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("pending producer real GET");
|
||||
assert_eq!(
|
||||
body,
|
||||
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
|
||||
"I05_HOOK_BYPASS: producer must enter its own scope"
|
||||
);
|
||||
self.ready
|
||||
.lock()
|
||||
.expect("pending signal")
|
||||
.take()
|
||||
.expect("one GET")
|
||||
.send(())
|
||||
.expect("producer entry signal");
|
||||
std::future::pending::<Result<GetObjectReader>>().await
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
panic!("observation must not PUT");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_hook_scope_restores_after_cancel() {
|
||||
let object = "config/observation-hook-cancel.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let adapter = Arc::new(PendingObservationStore {
|
||||
set: set.clone(),
|
||||
ready: std::sync::Mutex::new(Some(ready_tx)),
|
||||
});
|
||||
let mut scope = Box::pin(crate::config::com::read_config_observation(adapter, object, OBSERVATION_HOOK_BODY_LEN));
|
||||
tokio::select! {
|
||||
result = ready_rx => { result.expect("producer reached pending GET"); }
|
||||
result = &mut scope => { panic!("producer must remain pending until cancelled: {result:?}"); }
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
drop(scope);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
|
||||
@@ -16616,7 +16922,14 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
|
||||
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-versioned-delete";
|
||||
let object = "object.txt";
|
||||
let opts = ObjectOptions {
|
||||
@@ -16625,13 +16938,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"versioned object body".to_vec());
|
||||
let written = set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned object should be written");
|
||||
assert!(written.version_id.is_some());
|
||||
@@ -16948,7 +17274,14 @@ mod tests {
|
||||
/// `BucketNotEmpty` for a bucket the client has already emptied.
|
||||
#[tokio::test]
|
||||
async fn set_level_explicit_null_version_delete_purges_the_null_delete_marker() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-null-marker-purge";
|
||||
let object = "object.txt";
|
||||
// The delete path reads versioned/suspended from the bucket-config
|
||||
@@ -16970,13 +17303,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suspended-version object should be written");
|
||||
|
||||
@@ -17001,7 +17347,20 @@ mod tests {
|
||||
|
||||
let mut replacement_reader = PutObjReader::from_vec(b"replacement suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut replacement_reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut replacement_reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a suspended-version PUT should replace the first null marker");
|
||||
let replacement_marker = set_disks
|
||||
@@ -17061,7 +17420,14 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_level_version_suspended_delete_creates_null_delete_marker() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-version-suspended-delete";
|
||||
let object = "object.txt";
|
||||
let opts = ObjectOptions {
|
||||
@@ -17071,13 +17437,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suspended-version object should be written");
|
||||
|
||||
|
||||
@@ -113,6 +113,7 @@ pub(crate) struct DeleteRecoveryContext<'a> {
|
||||
pub repair_marker_source: bool,
|
||||
pub namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
|
||||
pub published_hint: Option<(Uuid, &'a [Result<DeleteRecoveryInspection>])>,
|
||||
pub can_commit: &'a (dyn Fn() -> bool + Sync),
|
||||
}
|
||||
|
||||
@@ -125,6 +126,7 @@ impl<'a> DeleteRecoveryContext<'a> {
|
||||
repair_marker_source: false,
|
||||
namespace_owner: None,
|
||||
recovery_ticket: None,
|
||||
published_hint: None,
|
||||
can_commit,
|
||||
}
|
||||
}
|
||||
@@ -421,7 +423,9 @@ impl SetDisks {
|
||||
let cleanup = loop {
|
||||
#[cfg(test)]
|
||||
test_hooks::record_cleanup_attempt(bucket, object);
|
||||
match self.recover_delete_key(bucket, object, context.clone()).await {
|
||||
let mut attempt_context = context.clone();
|
||||
attempt_context.published_hint = (attempt == 1).then_some((transaction, published.as_slice()));
|
||||
match self.recover_delete_key(bucket, object, attempt_context).await {
|
||||
// A stale settlement fingerprint only means the snapshot raced
|
||||
// another writer; each attempt re-inspects every participant.
|
||||
Err(DiskError::OutdatedXLMeta) if attempt < PUBLISHED_CLEANUP_ATTEMPTS && can_commit() => attempt += 1,
|
||||
@@ -551,6 +555,7 @@ impl SetDisks {
|
||||
explicit_delete,
|
||||
repair_marker_source,
|
||||
namespace_owner,
|
||||
published_hint,
|
||||
can_commit,
|
||||
recovery_ticket: _,
|
||||
} = context;
|
||||
@@ -589,7 +594,18 @@ impl SetDisks {
|
||||
let mut all_states = BTreeMap::new();
|
||||
for transaction in transactions {
|
||||
let mut complete = Vec::with_capacity(disks.len());
|
||||
for state in inspect(&disks, bucket, object, transaction).await {
|
||||
let inspections = if let Some((published_tx, published_states)) = published_hint
|
||||
&& !repair_marker_source
|
||||
&& !dry_run
|
||||
&& published_tx == transaction
|
||||
&& published_states.len() == disks.len()
|
||||
&& published_states.iter().all(Result::is_ok)
|
||||
{
|
||||
published_states.to_vec()
|
||||
} else {
|
||||
inspect(&disks, bucket, object, transaction).await
|
||||
};
|
||||
for state in inspections {
|
||||
match state {
|
||||
Ok(state) => {
|
||||
if let Some(journal) = &state.journal {
|
||||
@@ -832,6 +848,9 @@ impl SetDisks {
|
||||
bucket,
|
||||
copies,
|
||||
|copy| {
|
||||
if receipt(copy.state) && copy.commit_data_safe {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
if copy.state == DeleteRecoveryState::Absent
|
||||
|| (certified_same_transaction
|
||||
@@ -857,7 +876,10 @@ impl SetDisks {
|
||||
&disks,
|
||||
bucket,
|
||||
copies,
|
||||
|_| Some(DeleteRecoveryAction::FinishCommit),
|
||||
|copy| {
|
||||
(copy.state != DeleteRecoveryState::CommitFinished || !copy.commit_data_safe)
|
||||
.then_some(DeleteRecoveryAction::FinishCommit)
|
||||
},
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
@@ -922,13 +944,25 @@ impl SetDisks {
|
||||
// GC is a key-wide barrier: every predecessor must have settled
|
||||
// before the last surviving commit witness can disappear.
|
||||
let mut settled = Vec::with_capacity(all_states.len());
|
||||
for transaction in all_states.keys() {
|
||||
let fresh = inspect(&disks, bucket, object, *transaction).await;
|
||||
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
|
||||
if all_states.len() == 1 {
|
||||
let copies = all_states.into_values().next().ok_or(DiskError::FileCorrupt)?;
|
||||
if copies
|
||||
.iter()
|
||||
.any(|state| !matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))
|
||||
{
|
||||
summary.pending += 1;
|
||||
return Ok(summary);
|
||||
}
|
||||
settled.push(fresh.into_iter().collect::<Result<Vec<_>>>()?);
|
||||
settled.push(copies);
|
||||
} else {
|
||||
for transaction in all_states.keys() {
|
||||
let fresh = inspect(&disks, bucket, object, *transaction).await;
|
||||
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
|
||||
summary.pending += 1;
|
||||
return Ok(summary);
|
||||
}
|
||||
settled.push(fresh.into_iter().collect::<Result<Vec<_>>>()?);
|
||||
}
|
||||
}
|
||||
for (index, mut copies) in settled.into_iter().enumerate() {
|
||||
if index > 0 {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,7 +15,9 @@
|
||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
||||
use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
|
||||
use crate::set_disk::CompletePart;
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::{
|
||||
hermetic_set_disks_for_pool_with_default_parity_isolated, lifecycle_owned_put_options,
|
||||
};
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
@@ -39,18 +41,19 @@ async fn remote_metadata_heal_does_not_certify_unread_tier_payload() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
set.put_object(
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]),
|
||||
&ObjectOptions {
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("protected source");
|
||||
.await;
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]), &opts)
|
||||
.await
|
||||
.expect("protected source");
|
||||
for (disk, dir) in disks.iter().zip(&dirs) {
|
||||
let mut info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
@@ -94,10 +97,15 @@ async fn rollout_requires_both_flags_for_new_puts() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
for (requested, confirmed, protected) in [
|
||||
(false, false, false),
|
||||
@@ -159,12 +167,17 @@ async fn legacy_shard_repair_and_explicit_version_metadata_recovery_remain_avail
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let expected = vec![0x67; 1024 * 1024 + 37];
|
||||
for case in ["missing-part", "bad-bitrot", "missing-version-metadata"] {
|
||||
let object = case;
|
||||
@@ -272,10 +285,15 @@ async fn multipart_mode_survives_rollout_switch_changes() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
for protected in [false, true] {
|
||||
let object = format!("upload-{protected}");
|
||||
let upload =
|
||||
@@ -354,11 +372,16 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
|
||||
};
|
||||
let object = format!("source-{protected}");
|
||||
let expected = vec![0x39; 1024 * 1024 + 11];
|
||||
let seed_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
..Default::default()
|
||||
};
|
||||
let seed_opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let mut source = set
|
||||
.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &seed_opts)
|
||||
.await
|
||||
@@ -366,8 +389,8 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
|
||||
source.metadata_only = false;
|
||||
source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
shard_integrity_write_mode: None,
|
||||
..seed_opts.clone()
|
||||
};
|
||||
let copied = temp_env::async_with_vars(
|
||||
rollout_vars(!protected, !protected),
|
||||
|
||||
@@ -2247,7 +2247,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// lifetime; a fence by itself cannot transfer the outer lock lease.
|
||||
let detach_commit_owner = _upload_commit_guard.is_some()
|
||||
&& opts.namespace_lock_fence.is_none()
|
||||
&& (opts.bucket_lifecycle_lock_fence.is_none() || opts.bucket_lifecycle_lock_owner.is_some());
|
||||
&& (opts.bucket_lifecycle_lock_fence.is_none()
|
||||
|| opts.bucket_lifecycle_lock_owner.is_some()
|
||||
|| opts.bucket_incarnation_fence.is_some());
|
||||
let commit_set = self.clone();
|
||||
let commit_bucket = bucket.to_owned();
|
||||
let commit_object = object.to_owned();
|
||||
@@ -3902,7 +3904,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(write_quorum, None)
|
||||
.with_namespace_commit_guard(
|
||||
(!crate::bucket::utils::is_meta_bucketname(&commit_bucket))
|
||||
.then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
.then(|| commit_set.ctx.begin_namespace_commit() as Arc<dyn Send + Sync>),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
@@ -4179,14 +4181,12 @@ mod tests {
|
||||
use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test;
|
||||
use crate::set_disk::core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
// No-locker helpers resolve to the isolated-context variants (see
|
||||
// `hermetic_set_disks_isolated`); the guard-based tests build through
|
||||
// `hermetic_set_disks_with_lockers`, which stays on the bootstrap context
|
||||
// so their own ambient SetupTypeGuard flip remains visible to them.
|
||||
// All fixtures own isolated metadata contexts; lock-failure tests switch
|
||||
// the same context to distributed mode after seeding their objects.
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::{
|
||||
hermetic_set_disks_for_pool_with_default_parity_isolated as hermetic_set_disks_for_pool_with_default_parity,
|
||||
hermetic_set_disks_isolated as hermetic_set_disks, hermetic_set_disks_with_lockers,
|
||||
hermetic_set_disks_with_lockers_and_ctx,
|
||||
hermetic_set_disks_with_lockers_and_ctx, lifecycle_owned_put_options,
|
||||
};
|
||||
use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
@@ -4408,24 +4408,33 @@ mod tests {
|
||||
}
|
||||
|
||||
struct SetupTypeGuard {
|
||||
ctx: Arc<crate::runtime::instance::InstanceContext>,
|
||||
previous: SetupType,
|
||||
}
|
||||
|
||||
impl SetupTypeGuard {
|
||||
async fn switch_to(next: SetupType) -> Self {
|
||||
let previous = crate::runtime::sources::current_setup_type().await;
|
||||
crate::runtime::sources::set_setup_type(next).await;
|
||||
Self { previous }
|
||||
async fn switch_to(ctx: &Arc<crate::runtime::instance::InstanceContext>, next: SetupType) -> Self {
|
||||
let previous = if ctx.is_dist_erasure().await {
|
||||
SetupType::DistErasure
|
||||
} else {
|
||||
SetupType::Erasure
|
||||
};
|
||||
ctx.update_erasure_type(next).await;
|
||||
Self {
|
||||
ctx: Arc::clone(ctx),
|
||||
previous,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SetupTypeGuard {
|
||||
fn drop(&mut self) {
|
||||
let previous = self.previous.clone();
|
||||
let ctx = Arc::clone(&self.ctx);
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
std::thread::spawn(move || {
|
||||
handle.block_on(async {
|
||||
crate::runtime::sources::set_setup_type(previous).await;
|
||||
ctx.update_erasure_type(previous).await;
|
||||
});
|
||||
})
|
||||
.join()
|
||||
@@ -5397,10 +5406,15 @@ mod tests {
|
||||
bucket,
|
||||
object,
|
||||
&mut initial_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
&lifecycle_owned_put_options(
|
||||
&set_disks,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
.expect("initial fenced PUT should commit");
|
||||
@@ -5439,10 +5453,15 @@ mod tests {
|
||||
bucket,
|
||||
object,
|
||||
&mut winner_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
&lifecycle_owned_put_options(
|
||||
&set_disks,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
.expect("concurrent fenced PUT should advance the epoch");
|
||||
@@ -6185,15 +6204,22 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x47; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
@@ -6236,15 +6262,22 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn assert_abort_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x48; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let object_holder = set_disks
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
@@ -6300,17 +6333,365 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn assert_automatic_part_rollback(settlement_timeout: bool) {
|
||||
use crate::disk::{
|
||||
Disk,
|
||||
disk_store::LocalDiskWrapper,
|
||||
health_state::RuntimeDriveHealthState,
|
||||
os::{drain_namespace_for_test, prepared_publication_test_hooks as hooks},
|
||||
};
|
||||
use crate::io_support::shard_integrity::IntegrityBuilder;
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
use rustfs_filemeta::shard_integrity::IntegrityLayout;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("5")),
|
||||
(crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
|
||||
],
|
||||
async {
|
||||
for old in [false, true] {
|
||||
let (_roots, original_disks, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
|
||||
// This fixture normally disables health monitors. Install ordinary
|
||||
// monitored wrappers before any scenario I/O, on the same four disks.
|
||||
let disks = original_disks
|
||||
.iter()
|
||||
.map(|disk| match disk.as_ref() {
|
||||
Disk::Local(local) => Arc::new(Disk::Local(Box::new(LocalDiskWrapper::new(local.get_disk(), true)))),
|
||||
Disk::Remote(_) => unreachable!("four local disks"),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
*set_disks.disks.write().await = disks.iter().cloned().map(Some).collect();
|
||||
assert!(disks.iter().all(|disk| disk.health_check_enabled_for_test()));
|
||||
let bucket = "part-tail-rollback";
|
||||
let object = "object";
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
let (destination, completion) = if old {
|
||||
let payload = vec![0x41; 1 << 20];
|
||||
let create_opts = ObjectOptions {
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &create_opts).await;
|
||||
let (info, _) = set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload_id, false)
|
||||
.await
|
||||
.expect("acknowledged upload");
|
||||
(
|
||||
format!(
|
||||
"{}/{}/part.1",
|
||||
SetDisks::get_upload_id_dir(bucket, object, &upload_id),
|
||||
info.data_dir.expect("data directory")
|
||||
),
|
||||
Some((upload_id, parts, payload)),
|
||||
)
|
||||
} else {
|
||||
(format!("{}/part.1", Uuid::new_v4()), None)
|
||||
};
|
||||
let source = format!("{}/part.1", Uuid::new_v4());
|
||||
let mut builder =
|
||||
IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("index builder");
|
||||
builder.push([b"bbbb".as_slice(); 4].into_iter()).await.expect("stripe");
|
||||
let proof = builder.finish(8).await.expect("proof");
|
||||
let index = proof.inline_bytes().expect("inline index");
|
||||
let proof_name = proof.part.file_name();
|
||||
let meta = Bytes::from(
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 8,
|
||||
integrity: Some(proof.part),
|
||||
etag: "tail-retry".to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
.marshal_msg()
|
||||
.expect("metadata"),
|
||||
);
|
||||
let mut snapshots = Vec::new();
|
||||
let mut paths = Vec::new();
|
||||
for disk in &disks {
|
||||
let target = disk
|
||||
.get_object_path_for_io_if_local(RUSTFS_META_MULTIPART_BUCKET, &destination)
|
||||
.expect("local disk")
|
||||
.expect("target");
|
||||
let old_data = std::fs::read(&target).ok();
|
||||
let old_meta = std::fs::read(target.with_extension("1.meta")).ok();
|
||||
let old_index = old_meta
|
||||
.as_ref()
|
||||
.and_then(|bytes| ObjectPartInfo::unmarshal(bytes).ok())
|
||||
.and_then(|part| part.integrity)
|
||||
.map(|integrity| {
|
||||
let path = target.parent().expect("parent").join(integrity.file_name());
|
||||
let bytes = std::fs::read(&path).expect("acknowledged index");
|
||||
(path, bytes)
|
||||
});
|
||||
if old {
|
||||
assert!(old_index.is_some(), "acknowledged protected upload must have an integrity index");
|
||||
}
|
||||
snapshots.push((old_data, old_meta, old_index));
|
||||
paths.push(target);
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, &source, Bytes::from_static(b"bbbb"))
|
||||
.await
|
||||
.expect("retry shard");
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&format!("{}/{}", source.rsplit_once('/').expect("directory").0, proof_name),
|
||||
index.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("retry index");
|
||||
}
|
||||
let mut guards = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut entered = Vec::new();
|
||||
let mut settle_waiting = Vec::new();
|
||||
let mut settle_completed = Vec::new();
|
||||
let mut settle_timed_out = Vec::new();
|
||||
let mut restored = Vec::new();
|
||||
let mut finished = Vec::new();
|
||||
// Two of four publications cannot acknowledge the required quorum of three.
|
||||
for (number, target) in paths.iter().take(2).enumerate() {
|
||||
let paused_path = if number == 0 {
|
||||
target.clone()
|
||||
} else {
|
||||
target.with_extension("1.meta")
|
||||
};
|
||||
let transaction = target.parent().expect("parent").join(".part.1.rustfs-txn");
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let (release, blocked) = std::sync::mpsc::channel::<()>();
|
||||
guards.push(hooks::install_at(hooks::Stage::Rename, &paused_path, move || {
|
||||
let _ = tx.send(());
|
||||
let _ = blocked.recv();
|
||||
}));
|
||||
entered.push(rx);
|
||||
releases.push(release);
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
guards.push(hooks::install_at(hooks::Stage::NamespaceWaiting, target, move || {
|
||||
let _ = tx.send(());
|
||||
}));
|
||||
settle_waiting.push(rx);
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
guards.push(hooks::install_at(hooks::Stage::PartSettleTimeout, target, move || {
|
||||
let _ = tx.send(());
|
||||
}));
|
||||
settle_timed_out.push(rx);
|
||||
let restore = Arc::new(AtomicBool::new(false));
|
||||
let observed = restore.clone();
|
||||
guards.push(hooks::install_at(hooks::Stage::PartRestore, &transaction, move || {
|
||||
observed.store(true, Ordering::SeqCst);
|
||||
}));
|
||||
restored.push(restore);
|
||||
let complete = Arc::new(AtomicBool::new(false));
|
||||
let observed = complete.clone();
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
guards.push(hooks::install_at(hooks::Stage::PartSettleComplete, target, move || {
|
||||
observed.store(true, Ordering::SeqCst);
|
||||
let _ = tx.send(());
|
||||
}));
|
||||
settle_completed.push(rx);
|
||||
finished.push(complete);
|
||||
}
|
||||
let operation_set = set_disks.clone();
|
||||
let operation_disks = disks.iter().cloned().map(Some).collect::<Vec<_>>();
|
||||
let operation_source = source.clone();
|
||||
let operation_destination = destination.clone();
|
||||
let operation = tokio::spawn(async move {
|
||||
operation_set
|
||||
.rename_part(
|
||||
&operation_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&operation_source,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&operation_destination,
|
||||
meta,
|
||||
3,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
});
|
||||
let mut waited = Vec::new();
|
||||
let mut before_drain = Vec::new();
|
||||
let mut after_timeout = Vec::new();
|
||||
// Keep observations until both physical workers have drained. Even a
|
||||
// deliberately broken admission path must not strand a test worker.
|
||||
let observed = async {
|
||||
for receiver in entered {
|
||||
timeout(Duration::from_secs(30), receiver)
|
||||
.await
|
||||
.map_err(|_| "publisher did not reach barrier")?
|
||||
.map_err(|_| "publisher hook dropped")?;
|
||||
}
|
||||
for (waiting, completed) in settle_waiting.into_iter().zip(settle_completed) {
|
||||
let pending = timeout(Duration::from_secs(15), async {
|
||||
tokio::select! {
|
||||
result = waiting => result.map(|()| true),
|
||||
result = completed => result.map(|()| false),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| "automatic settlement neither waited nor completed")?
|
||||
.map_err(|_| "settlement observation dropped")?;
|
||||
waited.push(pending);
|
||||
}
|
||||
for (number, target) in paths.iter().take(2).enumerate() {
|
||||
let transaction = target.parent().expect("parent").join(".part.1.rustfs-txn");
|
||||
before_drain.push((
|
||||
restored[number].load(Ordering::SeqCst),
|
||||
finished[number].load(Ordering::SeqCst),
|
||||
transaction.join("new.meta").exists(),
|
||||
transaction.join("rollback").exists(),
|
||||
std::fs::read(target.with_extension("1.meta")).ok(),
|
||||
));
|
||||
}
|
||||
if settlement_timeout && waited.iter().all(|pending| *pending) {
|
||||
for receiver in settle_timed_out {
|
||||
timeout(Duration::from_secs(15), receiver)
|
||||
.await
|
||||
.map_err(|_| "settlement did not report Timeout")?
|
||||
.map_err(|_| "timeout observation dropped")?;
|
||||
}
|
||||
for (number, target) in paths.iter().take(2).enumerate() {
|
||||
let transaction = target.parent().expect("parent").join(".part.1.rustfs-txn");
|
||||
after_timeout.push((
|
||||
transaction.join("new.meta").exists(),
|
||||
transaction.join("rollback").exists(),
|
||||
restored[number].load(Ordering::SeqCst),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok::<(), &str>(())
|
||||
}
|
||||
.await;
|
||||
drop(releases);
|
||||
let mut drained = Vec::new();
|
||||
for target in paths.iter().take(2) {
|
||||
drained.push(timeout(Duration::from_secs(15), drain_namespace_for_test(target)).await);
|
||||
}
|
||||
let result = timeout(Duration::from_secs(30), operation).await;
|
||||
for result in drained {
|
||||
result.expect("actual publisher namespace drains");
|
||||
}
|
||||
observed.expect("observe automatic rollback under real deadlines");
|
||||
assert_eq!(waited, [true, true], "automatic rollback must really wait for the publisher's D lock");
|
||||
for (number, (restored, finished, has_meta, has_rollback, meta)) in before_drain.iter().enumerate() {
|
||||
assert!(!restored, "automatic rollback must not restore before the old publisher drains");
|
||||
assert!(!finished, "automatic rollback must wait for D, not return a competing-namespace error");
|
||||
assert!(*has_meta);
|
||||
assert!(!has_rollback, "no rollback write precedes physical drain");
|
||||
assert_eq!(meta, &snapshots[number].1);
|
||||
}
|
||||
if settlement_timeout {
|
||||
assert_eq!(after_timeout.len(), 2);
|
||||
for (has_meta, has_rollback, restored) in after_timeout {
|
||||
assert!(has_meta, "timed-out settlement retains recovery input");
|
||||
assert!(!has_rollback && !restored);
|
||||
}
|
||||
}
|
||||
let result = result.expect("automatic recovery returns").expect("quorum task");
|
||||
assert!(
|
||||
matches!(result, Err(DiskError::ErasureWriteQuorum)),
|
||||
"publication preserves its original quorum failure: {result:?}"
|
||||
);
|
||||
if settlement_timeout {
|
||||
// Two consecutive real timeouts take these drives offline. Only
|
||||
// production health probes may make them available to recovery.
|
||||
timeout(Duration::from_secs(30), async {
|
||||
let mut poll = tokio::time::interval(Duration::from_millis(25));
|
||||
while disks
|
||||
.iter()
|
||||
.any(|disk| disk.runtime_state() != RuntimeDriveHealthState::Online)
|
||||
{
|
||||
poll.tick().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("real recovery monitors restore all four disks");
|
||||
let directory = format!("{}/", destination.rsplit_once('/').expect("directory").0);
|
||||
set_disks
|
||||
.recover_part_transactions(&directory, 2, 3)
|
||||
.await
|
||||
.expect("existing recovery converges after monitor recovery");
|
||||
}
|
||||
// The ordinary case reaches these assertions directly from the
|
||||
// real rename_part return, with no supplemental recovery call.
|
||||
for (number, target) in paths.iter().enumerate() {
|
||||
assert_eq!(std::fs::read(target).ok(), snapshots[number].0, "data on disk {number}");
|
||||
assert_eq!(
|
||||
std::fs::read(target.with_extension("1.meta")).ok(),
|
||||
snapshots[number].1,
|
||||
"metadata on disk {number}"
|
||||
);
|
||||
if let Some((path, bytes)) = &snapshots[number].2 {
|
||||
assert_eq!(&std::fs::read(path).expect("old index survives"), bytes);
|
||||
}
|
||||
assert!(!target.parent().expect("parent").join(".part.1.rustfs-txn").exists());
|
||||
}
|
||||
if let Some((upload_id, parts, payload)) = completion {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("acknowledged old upload stays completable");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read restored object");
|
||||
assert_eq!(bytes, payload);
|
||||
}
|
||||
drop(guards);
|
||||
for disk in &disks {
|
||||
if let Disk::Local(local) = disk.as_ref() {
|
||||
local.stop_monitoring().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn legacy_part_mutation_flow_automatic_quorum_rollback_waits_for_tail() {
|
||||
assert_automatic_part_rollback(false).await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn legacy_part_mutation_flow_settlement_timeout_retains_recoverable_state() {
|
||||
assert_automatic_part_rollback(true).await;
|
||||
}
|
||||
|
||||
async fn assert_quorum_minus_one_retry_preserves_completable_part(
|
||||
disk_count: usize,
|
||||
parity: usize,
|
||||
success_indices: &[usize],
|
||||
) {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(disk_count, 0, parity).await;
|
||||
let bucket = format!("multipart-retry-{disk_count}-{}", success_indices[0]);
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, &bucket).await;
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(&bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
|
||||
let payload = vec![0x41; 1 << 20];
|
||||
let (upload_id, parts) =
|
||||
@@ -6489,12 +6870,18 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_upload_recovers_interrupted_part_retry() {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations as _;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(6, 0, 2).await;
|
||||
let bucket = "multipart-retry-recovery";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("multipart fixture metadata owner")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("multipart fixture bucket has authoritative quota metadata");
|
||||
|
||||
let payload = vec![0x42; 1 << 20];
|
||||
let (upload_id, parts) =
|
||||
@@ -6654,10 +7041,15 @@ mod tests {
|
||||
bucket,
|
||||
object,
|
||||
&mut initial_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
&lifecycle_owned_put_options(
|
||||
&set_disks,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
.expect("initial object should be written");
|
||||
@@ -7068,7 +7460,7 @@ mod tests {
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let holder = set_disks
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await
|
||||
@@ -7104,7 +7496,7 @@ mod tests {
|
||||
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
|
||||
Err(StorageError::InvalidUploadID(..))
|
||||
));
|
||||
for temp_dir in temp_dirs {
|
||||
for temp_dir in &temp_dirs {
|
||||
assert!(
|
||||
!temp_dir
|
||||
.path()
|
||||
@@ -7133,7 +7525,7 @@ mod tests {
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartAfterRename);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -7170,9 +7562,11 @@ mod tests {
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn cancelled_put_object_part_keeps_upload_lock_through_commit() {
|
||||
for pause in [
|
||||
MultipartCommitPause::PutPartAfterCapacityAdmission,
|
||||
MultipartCommitPause::PutPartAfterRename,
|
||||
for (pause, typed_owner) in [
|
||||
(MultipartCommitPause::PutPartAfterCapacityAdmission, false),
|
||||
(MultipartCommitPause::PutPartAfterCapacityAdmission, true),
|
||||
(MultipartCommitPause::PutPartAfterRename, false),
|
||||
(MultipartCommitPause::PutPartAfterRename, true),
|
||||
] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
@@ -7188,7 +7582,18 @@ mod tests {
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let put_opts = if typed_owner {
|
||||
lifecycle_owned_put_options(&set_disks, bucket, ObjectOptions::default()).await
|
||||
} else {
|
||||
ObjectOptions::default()
|
||||
};
|
||||
let lifecycle_owner = put_opts.bucket_incarnation_fence.as_ref().map(Arc::downgrade);
|
||||
if typed_owner {
|
||||
assert!(put_opts.bucket_lifecycle_lock_fence.is_some());
|
||||
assert!(put_opts.bucket_lifecycle_lock_owner.is_none());
|
||||
assert!(lifecycle_owner.is_some());
|
||||
}
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, pause);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -7196,7 +7601,7 @@ mod tests {
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
|
||||
put_store
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &put_opts)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
@@ -7207,6 +7612,13 @@ mod tests {
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
if let Some(owner) = lifecycle_owner {
|
||||
assert!(
|
||||
owner.upgrade().is_some(),
|
||||
"cancelled caller must not release the commit's typed lifecycle owner"
|
||||
);
|
||||
}
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let mut abort = tokio::spawn(async move {
|
||||
@@ -7256,7 +7668,7 @@ mod tests {
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
// issue#5961: the barrier releases only once BOTH commits are paused
|
||||
// inside their commit sections, so reaching wait_until_paused proves the
|
||||
// two part numbers held their commit locks concurrently. Under an
|
||||
@@ -7352,7 +7764,7 @@ mod tests {
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_lock_path = format!("{upload_id_path}/part.1");
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartAfterRename);
|
||||
|
||||
let first_store = set_disks.clone();
|
||||
@@ -7461,7 +7873,7 @@ mod tests {
|
||||
let part_lock_path = format!("{upload_id_path}/part.1");
|
||||
*target.write().expect("lock-loss target should be writable") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, part_lock_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -7529,7 +7941,7 @@ mod tests {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
*target.write().expect("lock-loss target should be writable") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -7774,6 +8186,92 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_path_delete_flow_multipart_cleanup_preserves_order_and_quorum() {
|
||||
use crate::storage_api_contracts::bucket::BucketOperations;
|
||||
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
for disk in &disks {
|
||||
for path in ["cleanup/first", "cleanup/blocker", "cleanup/last"] {
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, path, Bytes::from_static(b"staged"))
|
||||
.await
|
||||
.expect("seed raw multipart file");
|
||||
}
|
||||
}
|
||||
set.cleanup_multipart_path(&[
|
||||
"cleanup/first".to_owned(),
|
||||
"cleanup/blocker/child".to_owned(),
|
||||
"cleanup/last".to_owned(),
|
||||
])
|
||||
.await;
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, "cleanup/first").await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"batch keeps the successful item before its first error"
|
||||
);
|
||||
for path in ["cleanup/blocker", "cleanup/last"] {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, path)
|
||||
.await
|
||||
.expect("later file remains"),
|
||||
b"staged".as_slice(),
|
||||
"batch must stop at the first non-directory error"
|
||||
);
|
||||
}
|
||||
}
|
||||
set.cleanup_multipart_path(&["cleanup/blocker".to_owned(), "cleanup/last".to_owned()])
|
||||
.await;
|
||||
for disk in &disks {
|
||||
for path in ["cleanup/blocker", "cleanup/last"] {
|
||||
assert!(
|
||||
matches!(disk.read_all(RUSTFS_META_MULTIPART_BUCKET, path).await, Err(DiskError::FileNotFound)),
|
||||
"retry cleans remaining raw files on every disk"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let bucket = "path-delete-multipart";
|
||||
crate::bucket::metadata_sys::object_store_in(&set.ctx)
|
||||
.await
|
||||
.expect("fixture object store")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("authoritative multipart bucket");
|
||||
for available in [3, 2] {
|
||||
let object = format!("object-{available}");
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload before reducing availability");
|
||||
let saved = {
|
||||
let mut current = set.disks.write().await;
|
||||
let saved = current.clone();
|
||||
for disk in current.iter_mut().skip(available) {
|
||||
*disk = None;
|
||||
}
|
||||
saved
|
||||
};
|
||||
let result = set
|
||||
.abort_multipart_upload(bucket, &object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await;
|
||||
*set.disks.write().await = saved;
|
||||
let remaining = set.check_upload_id_exists(bucket, &object, &upload.upload_id, false).await;
|
||||
if available == 3 {
|
||||
result.expect("three of four disks satisfy delete write quorum");
|
||||
assert!(matches!(remaining, Err(StorageError::InvalidUploadID(..))));
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(StorageError::ErasureWriteQuorum)),
|
||||
"two disks must not acknowledge an abort"
|
||||
);
|
||||
remaining.expect("failed abort leaves quorum-visible staging after disks return");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_metadata_io_does_not_block_disk_registry_updates() {
|
||||
use crate::set_disk::core::io_primitives::rename_fanout_barrier::{self, PHASE_READ_VERSION};
|
||||
@@ -7940,7 +8438,7 @@ mod tests {
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x42; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let holder = set_disks
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await
|
||||
@@ -8025,7 +8523,7 @@ mod tests {
|
||||
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
|
||||
signaling.set_target(upload_resource.clone());
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
@@ -8165,7 +8663,7 @@ mod tests {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
@@ -8270,7 +8768,7 @@ mod tests {
|
||||
use futures::FutureExt;
|
||||
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async {
|
||||
let (dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let (mut dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-physical-namespace";
|
||||
let object = if undo { "undo-tail" } else { "publication-tail" };
|
||||
let old_body = vec![0x41; 1024];
|
||||
@@ -8440,7 +8938,7 @@ mod tests {
|
||||
if !coordinator_drained || !physical_drained || owner_drained.is_err() {
|
||||
// A bounded cleanup failure cannot justify deleting roots that a
|
||||
// detached executor might still use. Keep them for diagnosis.
|
||||
let retained = dirs.into_iter().map(TempDir::keep).collect::<Vec<_>>();
|
||||
let retained = dirs.keep_dirs();
|
||||
eprintln!("MPU cleanup incomplete: coordinator={coordinator_drained}, physical={physical_drained}, retained={retained:?}");
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
@@ -8611,7 +9109,7 @@ mod tests {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let completion_barrier =
|
||||
MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
|
||||
|
||||
@@ -8722,7 +9220,7 @@ mod tests {
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x65; 4096], &ObjectOptions::default()).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let completion_barrier =
|
||||
MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeObjectPublication);
|
||||
|
||||
@@ -8778,7 +9276,7 @@ mod tests {
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x49; 4096], &ObjectOptions::default()).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
@@ -8845,7 +9343,7 @@ mod tests {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
*target.write().expect("lock-loss target should be writable") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(&set_disks.ctx, SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeLockLost);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
|
||||
+1505
-1008
File diff suppressed because it is too large
Load Diff
@@ -16,7 +16,7 @@
|
||||
//! another object (`a/b` and `a/b/c`). Cleanup of the parent must only remove
|
||||
//! the parent's own protocol data and never touch the child's xl.meta or data.
|
||||
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use super::hermetic_set_disks_support::{HermeticSetFixture, hermetic_set_disks_isolated};
|
||||
use super::*;
|
||||
use crate::data_movement::{SourceCleanupBucketFence, SourceCleanupError, cleanup_source_entry_if_unchanged};
|
||||
use rustfs_filemeta::FileInfoVersions;
|
||||
@@ -100,7 +100,7 @@ fn child_tree(temp_dirs: &[TempDir], bucket: &str) -> Vec<Vec<String>> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn setup(payload_len: usize) -> (Vec<TempDir>, Arc<SetDisks>, &'static str, Vec<u8>, Vec<u8>) {
|
||||
async fn setup(payload_len: usize) -> (HermeticSetFixture, Arc<SetDisks>, &'static str, Vec<u8>, Vec<u8>) {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "data-movement-prefix-cleanup";
|
||||
for disk in &disk_stores {
|
||||
|
||||
@@ -3885,8 +3885,7 @@ mod metadata_cache_tests {
|
||||
|
||||
let bucket = "late-metadata-identity";
|
||||
let object = "late-metadata-identity-object";
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await;
|
||||
let (_fixture, _disks, set) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create real late-read bucket");
|
||||
|
||||
@@ -8,6 +8,7 @@ pub(crate) mod admin {
|
||||
}
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use rustfs_storage_api::bucket::valid_artifact_base_prefix;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
BUCKET_LIFECYCLE_LOCK_OBJECT, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
SRBucketDeleteOp,
|
||||
|
||||
+274
-233
@@ -848,255 +848,293 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||
if !is_meta_bucketname(bucket)
|
||||
&& let Err(err) = check_valid_bucket_name_strict(bucket)
|
||||
{
|
||||
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
||||
}
|
||||
self.handle_make_bucket_inner(bucket, opts).await
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace.
|
||||
let bucket_lifecycle_guard = if !opts.no_lock {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
|
||||
Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
|
||||
Some(
|
||||
await_bucket_namespace_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket namespace lock acquisition",
|
||||
async {
|
||||
ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
|
||||
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: bucket.to_string(),
|
||||
object: bucket.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
}
|
||||
}
|
||||
other => StorageError::Lock(other),
|
||||
})
|
||||
},
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) => {
|
||||
if is_err_bucket_not_found(&err) {
|
||||
None
|
||||
} else {
|
||||
return Err(to_object_err(err, vec![bucket]));
|
||||
}
|
||||
// Keep the large creation future out of the instrumented handler's poll
|
||||
// frame. Guards remain owned by the same task.
|
||||
fn handle_make_bucket_inner<'a>(
|
||||
&'a self,
|
||||
bucket: &'a str,
|
||||
opts: &'a MakeBucketOptions,
|
||||
) -> futures::future::BoxFuture<'a, Result<()>> {
|
||||
Box::pin(async move {
|
||||
if opts.artifact_base_prefix.as_deref().is_some_and(|prefix| {
|
||||
is_meta_bucketname(bucket)
|
||||
|| opts.no_lock
|
||||
|| !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(prefix)
|
||||
}) {
|
||||
return Err(Error::other("invalid artifact bucket creation options"));
|
||||
}
|
||||
};
|
||||
let confirmed_missing = existing_bucket_info.is_none();
|
||||
let mut pending_creation_metadata = None;
|
||||
if confirmed_missing
|
||||
&& !opts.no_lock
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& let Some(metadata_store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await
|
||||
&& let Some(old_incarnation) = crate::bucket::metadata::load_bucket_incarnation(metadata_store, bucket).await?
|
||||
{
|
||||
// A quorum-level absence is not proof that every disk lost the old
|
||||
// generation. Require the explicit recovery path to verify all disks
|
||||
// and retire this incarnation before a same-name create can mutate
|
||||
// any volume.
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
StorageError::BucketExists(_) => StorageError::InvalidArgument(
|
||||
if !is_meta_bucketname(bucket)
|
||||
&& let Err(err) = check_valid_bucket_name_strict(bucket)
|
||||
{
|
||||
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace.
|
||||
let bucket_lifecycle_guard = if !opts.no_lock {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
|
||||
Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
|
||||
Some(
|
||||
await_bucket_namespace_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket namespace lock acquisition",
|
||||
async {
|
||||
ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
|
||||
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: bucket.to_string(),
|
||||
object: bucket.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
}
|
||||
}
|
||||
other => StorageError::Lock(other),
|
||||
})
|
||||
},
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) => {
|
||||
if is_err_bucket_not_found(&err) {
|
||||
None
|
||||
} else {
|
||||
return Err(to_object_err(err, vec![bucket]));
|
||||
}
|
||||
}
|
||||
};
|
||||
let confirmed_missing = existing_bucket_info.is_none();
|
||||
let mut pending_creation_metadata = None;
|
||||
if confirmed_missing
|
||||
&& !opts.no_lock
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& let Some(metadata_store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await
|
||||
&& let Some(old_incarnation) = crate::bucket::metadata::load_bucket_incarnation(metadata_store, bucket).await?
|
||||
{
|
||||
// A quorum-level absence is not proof that every disk lost the old
|
||||
// generation. Require the explicit recovery path to verify all disks
|
||||
// and retire this incarnation before a same-name create can mutate
|
||||
// any volume.
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
StorageError::BucketExists(_) => StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
"an old bucket volume remains on disk; administrator recovery refused".to_string(),
|
||||
),
|
||||
other => other,
|
||||
})?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let pending_intent = persisted
|
||||
&& metadata.bucket_incarnation_id == old_incarnation
|
||||
&& metadata.needs_bucket_creation_commit()
|
||||
&& !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
|
||||
if pending_intent {
|
||||
pending_creation_metadata = Some(metadata);
|
||||
} else {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
"an old bucket volume remains on disk; administrator recovery refused".to_string(),
|
||||
),
|
||||
other => other,
|
||||
})?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let pending_intent = persisted
|
||||
&& metadata.bucket_incarnation_id == old_incarnation
|
||||
&& metadata.needs_bucket_creation_commit()
|
||||
&& !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
|
||||
if pending_intent {
|
||||
pending_creation_metadata = Some(metadata);
|
||||
} else {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
|
||||
));
|
||||
}
|
||||
}
|
||||
let existing_metadata = if let Some(metadata) = pending_creation_metadata {
|
||||
Some(metadata)
|
||||
} else if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
metadata.created = existing_bucket_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.created)
|
||||
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// A sidecar left by an interrupted legacy migration is the
|
||||
// bucket's published incarnation; keep it rather than minting
|
||||
// one that would invalidate fences taken on the stored value.
|
||||
// A retired incarnation is residue of a deleted bucket and
|
||||
// must not come back.
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
|
||||
));
|
||||
}
|
||||
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
} else if !metadata.bucket_incarnation_sidecar {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
Some(metadata)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| {
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
BucketMetadata::new_with_default_durability(bucket)
|
||||
let existing_metadata = if let Some(metadata) = pending_creation_metadata {
|
||||
Some(metadata)
|
||||
} else if (opts.force_create || opts.artifact_base_prefix.is_some())
|
||||
&& !confirmed_missing
|
||||
&& !is_meta_bucketname(bucket)
|
||||
{
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
metadata.created = existing_bucket_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.created)
|
||||
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// A sidecar left by an interrupted legacy migration is the
|
||||
// bucket's published incarnation; keep it rather than minting
|
||||
// one that would invalidate fences taken on the stored value.
|
||||
// A retired incarnation is residue of a deleted bucket and
|
||||
// must not come back.
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
}
|
||||
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
} else if !metadata.bucket_incarnation_sidecar {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
Some(metadata)
|
||||
} else {
|
||||
BucketMetadata::new(bucket)
|
||||
}
|
||||
});
|
||||
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
|
||||
if confirmed_missing || is_meta_bucketname(bucket) {
|
||||
meta.set_created(opts.created_at);
|
||||
None
|
||||
};
|
||||
|
||||
if opts.versioning_enabled {
|
||||
let has_creation_identity = existing_metadata.is_some();
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| {
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
BucketMetadata::new_with_default_durability(bucket)
|
||||
} else {
|
||||
BucketMetadata::new(bucket)
|
||||
}
|
||||
});
|
||||
if let Some(prefix) = &opts.artifact_base_prefix {
|
||||
if has_creation_identity {
|
||||
if meta.artifact_base_prefix.as_ref() != Some(prefix) {
|
||||
return Err(Error::other("artifact namespace is immutable"));
|
||||
}
|
||||
} else if confirmed_missing {
|
||||
meta.artifact_base_prefix = Some(prefix.clone());
|
||||
} else {
|
||||
return Err(Error::other("artifact namespace requires a new bucket"));
|
||||
}
|
||||
}
|
||||
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
|
||||
if confirmed_missing || is_meta_bucketname(bucket) {
|
||||
meta.set_created(opts.created_at);
|
||||
|
||||
if opts.versioning_enabled || meta.artifact_base_prefix.is_some() {
|
||||
meta.versioning_config_xml =
|
||||
crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Object Lock enable is one-way, and it must apply to an existing bucket
|
||||
// too. Site replication replays make-with-versioning carrying the
|
||||
// source's lockEnabled against a destination bucket that already exists;
|
||||
// gating this on `confirmed_missing` returned success while leaving the
|
||||
// replica unlocked, so replicated versions could be deleted without the
|
||||
// retention the source enforces.
|
||||
let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
|
||||
if opts.lock_enabled {
|
||||
meta.lock_enabled = true;
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml =
|
||||
crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Object Lock enable is one-way, and it must apply to an existing bucket
|
||||
// too. Site replication replays make-with-versioning carrying the
|
||||
// source's lockEnabled against a destination bucket that already exists;
|
||||
// gating this on `confirmed_missing` returned success while leaving the
|
||||
// replica unlocked, so replicated versions could be deleted without the
|
||||
// retention the source enforces.
|
||||
let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
|
||||
if opts.lock_enabled {
|
||||
meta.lock_enabled = true;
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
|
||||
let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled;
|
||||
if metadata_persisted_before_physical {
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
|
||||
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
|
||||
}
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// Fence every scanner cycle that could have observed the namespace
|
||||
// before physical creation. Creation may become visible even when a
|
||||
// later metadata write or namespace-lock check fails.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
|
||||
}
|
||||
|
||||
if let Err(err) = await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"physical bucket creation",
|
||||
await_bucket_namespace_operation(
|
||||
metadata_transaction_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket creation metadata transaction",
|
||||
async {
|
||||
self.make_bucket_on_sets(bucket, opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if is_err_bucket_exists(&err)
|
||||
&& let Err(heal_err) = self
|
||||
.handle_heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
|
||||
}
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
|
||||
error!("make bucket failed: {err}");
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
};
|
||||
|
||||
let metadata_result = async {
|
||||
let metadata_persisted_before_physical =
|
||||
confirmed_missing && !is_meta_bucketname(bucket) && (opts.lock_enabled || meta.artifact_base_prefix.is_some());
|
||||
if metadata_persisted_before_physical {
|
||||
return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
|
||||
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
|
||||
}
|
||||
}
|
||||
if is_meta_bucketname(bucket) {
|
||||
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else if existing_incarnation_is_authoritative && !lock_newly_enabled {
|
||||
metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else {
|
||||
metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
if let Err(err) = metadata_result {
|
||||
if !metadata_lock_lost {
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
if metadata_lock_lost {
|
||||
return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// A scanner may have sampled the first fence before the bucket
|
||||
// became visible. Fence again after metadata initialization so
|
||||
// that snapshot cannot publish as complete.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// Fence every scanner cycle that could have observed the namespace
|
||||
// before physical creation. Creation may become visible even when a
|
||||
// later metadata write or namespace-lock check fails.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
// Keep cleanup state out of the bucket-creation request future.
|
||||
Box::pin(self.cleanup_bucket_usage(bucket, ns_guard.as_ref())).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
if let Err(err) = await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"physical bucket creation",
|
||||
await_bucket_namespace_operation(
|
||||
metadata_transaction_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket creation metadata transaction",
|
||||
async {
|
||||
self.make_bucket_on_sets(bucket, opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if is_err_bucket_exists(&err)
|
||||
&& let Err(heal_err) = self
|
||||
.handle_heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
|
||||
}
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost())
|
||||
{
|
||||
error!("make bucket failed: {err}");
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
};
|
||||
|
||||
let metadata_result = async {
|
||||
if metadata_persisted_before_physical {
|
||||
return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
|
||||
}
|
||||
if is_meta_bucketname(bucket) {
|
||||
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else if existing_incarnation_is_authoritative && !lock_newly_enabled {
|
||||
metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else {
|
||||
metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
if let Err(err) = metadata_result {
|
||||
if !metadata_lock_lost {
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
if metadata_lock_lost {
|
||||
return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// A scanner may have sampled the first fence before the bucket
|
||||
// became visible. Fence again after metadata initialization so
|
||||
// that snapshot cannot publish as complete.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Prove a live bucket generation before repairing missing expansion volumes.
|
||||
@@ -1461,6 +1499,9 @@ impl ECStore {
|
||||
}
|
||||
};
|
||||
if bucket_exists {
|
||||
if !is_meta_bucketname(bucket) {
|
||||
self.require_no_artifact_backup_retention(bucket).await?;
|
||||
}
|
||||
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
|
||||
|
||||
if !opts.force {
|
||||
|
||||
@@ -160,6 +160,7 @@ pub struct BucketIncarnationFenceGuard {
|
||||
bucket: String,
|
||||
token: Option<u64>,
|
||||
memoized: Option<Uuid>,
|
||||
validated_incarnation: Option<Uuid>,
|
||||
}
|
||||
|
||||
impl BucketIncarnationFenceGuard {
|
||||
@@ -173,6 +174,7 @@ impl BucketIncarnationFenceGuard {
|
||||
bucket: bucket.to_owned(),
|
||||
token,
|
||||
memoized,
|
||||
validated_incarnation: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,6 +185,7 @@ impl BucketIncarnationFenceGuard {
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<Uuid>>,
|
||||
{
|
||||
self.validated_incarnation = None;
|
||||
let cacheable = reuse_memo && self.token.is_some();
|
||||
let current = match self.memoized.filter(|_| cacheable) {
|
||||
Some(current) => current,
|
||||
@@ -197,6 +200,7 @@ impl BucketIncarnationFenceGuard {
|
||||
}
|
||||
};
|
||||
self.ensure_held()?;
|
||||
self.validated_incarnation = Some(current);
|
||||
Ok(current)
|
||||
}
|
||||
|
||||
@@ -213,6 +217,19 @@ impl BucketIncarnationFenceGuard {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, store: &super::ECStore, bucket: &str, incarnation: Uuid) -> bool {
|
||||
Arc::ptr_eq(&self.registry, &store.bucket_fence_registry)
|
||||
&& self.bucket == bucket
|
||||
&& self.validated_incarnation == Some(incarnation)
|
||||
&& !self.is_lock_lost()
|
||||
}
|
||||
|
||||
/// Retain this validated lifecycle owner through the storage write tail.
|
||||
pub fn attach_owner(self: &Arc<Self>, opts: &mut crate::object_api::ObjectOptions) {
|
||||
self.attach_to_object_options(opts);
|
||||
opts.bucket_incarnation_fence = Some(Arc::clone(self));
|
||||
}
|
||||
|
||||
/// Propagate lifecycle lock loss into the storage commit checks.
|
||||
/// The caller still owns this guard until the complete write tail drains.
|
||||
pub fn attach_to_object_options(&self, opts: &mut crate::object_api::ObjectOptions) {
|
||||
@@ -221,7 +238,8 @@ impl BucketIncarnationFenceGuard {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
/// Report whether the lifecycle lock no longer covers this operation.
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
}
|
||||
|
||||
@@ -401,6 +419,30 @@ mod tests {
|
||||
BucketIncarnationFenceGuard::new(registry, "b", inner, cacheable)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_owner_tracks_validation_instead_of_cached_incarnation() {
|
||||
let registry = Arc::new(BucketFenceRegistry::default());
|
||||
let lock = NamespaceLock::new("artifact-owner-validation".to_owned(), Arc::new(LocalClient::new()));
|
||||
let mut first = test_fence(registry.clone(), &lock, true).await;
|
||||
first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo");
|
||||
let mut snapshot = test_fence(registry, &lock, true).await;
|
||||
assert_eq!(snapshot.memoized, Some(uuid(7)));
|
||||
assert_eq!(snapshot.validated_incarnation, None, "registration alone grants no artifact ownership");
|
||||
snapshot
|
||||
.validate(false, || async { Ok(uuid(8)) })
|
||||
.await
|
||||
.expect("validate snapshot");
|
||||
assert_eq!(snapshot.validated_incarnation, Some(uuid(8)));
|
||||
assert_eq!(snapshot.memoized, Some(uuid(7)), "independent snapshots do not update the shared memo");
|
||||
assert!(
|
||||
snapshot
|
||||
.validate(false, || async { Err(StorageError::ErasureReadQuorum) })
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(snapshot.validated_incarnation, None, "failed validation cannot retain earlier ownership");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overlapping_validations_reuse_one_success_until_coverage_gap() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
@@ -39,6 +39,10 @@ pub(crate) struct BucketHealScope {
|
||||
}
|
||||
|
||||
impl BucketHealScope {
|
||||
pub(crate) fn lifecycle_guard(&self) -> Option<&NamespaceLockGuard> {
|
||||
self.fence.namespace_lock_guard()
|
||||
}
|
||||
|
||||
pub(crate) fn check(&self) -> disk::error::Result<()> {
|
||||
if self.fence.is_lock_lost() {
|
||||
return Err(DiskError::other("bucket heal incarnation fence was lost"));
|
||||
@@ -1070,16 +1074,25 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.handle_heal_object_with_absence(
|
||||
crate::bucket::artifact::with_artifact_repair_scope(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut None,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
Box::pin(async {
|
||||
self.handle_heal_object_with_absence(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut None,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1113,6 +1126,24 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_unversioned_absence: bool,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
crate::bucket::artifact::with_artifact_repair_scope(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
Box::pin(self.heal_object_with_absence_proof_inner(bucket, object, version_id, opts, allow_unversioned_absence)),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_absence_proof_inner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_unversioned_absence: bool,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
// Object heal may recreate a missing bucket volume before committing a
|
||||
// shard. Keep the current bucket generation fenced for the whole
|
||||
@@ -1131,7 +1162,11 @@ impl ECStore {
|
||||
// Reuse the admission's owner: reacquiring a read lock behind a queued
|
||||
// lifecycle writer would deadlock with the read lock we already hold.
|
||||
let scope = bucket_heal_scope(bucket).filter(|scope| std::ptr::eq(scope.store.as_ref(), self));
|
||||
let guard = if let Some(scope) = &scope {
|
||||
let repair = crate::bucket::artifact::artifact_repair_scope(bucket, object);
|
||||
let guard = if let Some(repair) = &repair {
|
||||
repair.check()?;
|
||||
None
|
||||
} else if let Some(scope) = &scope {
|
||||
scope.check().map_err(Error::from)?;
|
||||
None
|
||||
} else {
|
||||
@@ -1139,11 +1174,13 @@ impl ECStore {
|
||||
};
|
||||
let fence_is_lost = || {
|
||||
scope.as_ref().is_some_and(|scope| scope.fence.is_lock_lost())
|
||||
|| repair.as_ref().is_some_and(|scope| scope.check().is_err())
|
||||
|| guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
};
|
||||
let lifecycle_guard = scope
|
||||
.as_ref()
|
||||
.and_then(|scope| scope.fence.namespace_lock_guard())
|
||||
.or_else(|| repair.as_ref().and_then(|scope| scope.lifecycle_guard()))
|
||||
.or(guard.as_ref())
|
||||
.ok_or_else(|| Error::other("bucket heal requires a held lifecycle guard"))?;
|
||||
let retirement = crate::bucket::retirement::MarkerRetirementContext {
|
||||
@@ -2045,6 +2082,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4148,6 +4186,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
};
|
||||
|
||||
let err = store
|
||||
|
||||
@@ -428,6 +428,7 @@ impl ECStore {
|
||||
|
||||
let mut deployment_id = None;
|
||||
let mut pool_meta_bootstrap_authorities = Vec::new();
|
||||
let mut pending_format_disks = Vec::new();
|
||||
|
||||
// let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?;
|
||||
|
||||
@@ -553,6 +554,7 @@ impl ECStore {
|
||||
// formatted it, `LegacyAdoption` when it verified the migration, and
|
||||
// `None` when it merely read a format another node created.
|
||||
pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
|
||||
pending_format_disks.extend(loaded_format.pending_disks);
|
||||
let fm = loaded_format.format;
|
||||
|
||||
// Format loading succeeded, enable health monitoring on all disks
|
||||
@@ -618,6 +620,7 @@ impl ECStore {
|
||||
// this point) and later reads share one cell.
|
||||
ctx: instance_ctx.clone(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
|
||||
// Only set it when this instance's deployment ID is not yet configured
|
||||
@@ -627,8 +630,21 @@ impl ECStore {
|
||||
|
||||
let wait_sec = 5;
|
||||
let mut exit_count = 0;
|
||||
let disk_sets = ec
|
||||
.pools
|
||||
.iter()
|
||||
.flat_map(|pool| pool.disk_set.iter().cloned())
|
||||
.collect::<Vec<_>>();
|
||||
loop {
|
||||
if let Err(err) = ec.init(ctx.clone()).await {
|
||||
let initialization = async {
|
||||
ec.init(ctx.clone()).await?;
|
||||
for pending in &pending_format_disks {
|
||||
pending.finalize(&instance_ctx, &disk_sets).await?;
|
||||
}
|
||||
Ok::<(), Error>(())
|
||||
}
|
||||
.await;
|
||||
if let Err(err) = initialization {
|
||||
error!("init err: {}", err);
|
||||
error!("retry after {} second", wait_sec);
|
||||
sleep(Duration::from_secs(wait_sec)).await;
|
||||
@@ -652,6 +668,11 @@ impl ECStore {
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
// Keep startup state out of callers and the instrumented wrapper.
|
||||
Box::pin(self.init_inner(rx)).await
|
||||
}
|
||||
|
||||
async fn init_inner(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let should_persist_pool_meta = self
|
||||
@@ -869,6 +890,23 @@ impl ECStore {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_bucket_tests;
|
||||
#[cfg(unix)]
|
||||
mod artifact_inventory_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_put_seal_tests;
|
||||
#[cfg(all(feature = "test-util", unix))]
|
||||
mod artifact_receiver_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_repair_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_retirement_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_write_entrypoint_tests;
|
||||
#[cfg(all(feature = "test-util", unix))]
|
||||
mod artifact_writes_seal_tests;
|
||||
|
||||
use super::{
|
||||
LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
|
||||
establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
|
||||
@@ -1781,7 +1819,7 @@ mod tests {
|
||||
let mut pools = Vec::new();
|
||||
for _ in 0..4 {
|
||||
let (pool_dirs, pool_disks, pool) = hermetic_set_disks_isolated(4).await;
|
||||
dirs.extend(pool_dirs);
|
||||
dirs.push(pool_dirs);
|
||||
disks.push(pool_disks);
|
||||
pools.push(pool);
|
||||
}
|
||||
@@ -7298,8 +7336,8 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("seed older target metadata on all disks");
|
||||
let source = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let mut source = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("read source marker metadata")
|
||||
@@ -7308,6 +7346,30 @@ mod tests {
|
||||
.into_iter()
|
||||
.find(|version| version.version_id.unwrap_or_default() == marker_id)
|
||||
.expect("find source delete marker");
|
||||
if marker_incarnation.is_none() {
|
||||
// Public writes bind new markers to the current bucket generation.
|
||||
// Seed legacy disk metadata explicitly without weakening admission.
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut source.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_BUCKET_INCARNATION_ID,
|
||||
);
|
||||
for disk in source_set.disk_inventory().await.into_iter().flatten() {
|
||||
disk.write_metadata("", &bucket, object, source.clone())
|
||||
.await
|
||||
.expect("seed legacy source marker metadata");
|
||||
}
|
||||
source_set.invalidate_get_object_metadata_cache(&bucket, object).await;
|
||||
}
|
||||
let source = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("reload source marker metadata")
|
||||
.expect("source marker must remain present")
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.version_id.unwrap_or_default() == marker_id)
|
||||
.expect("find reloaded source delete marker");
|
||||
assert_eq!(source.delete_marker_incarnation(), marker_incarnation, "source fixture identity");
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
for disk in target_set.disk_inventory().await.into_iter().flatten().take(2) {
|
||||
disk.write_metadata("", &bucket, object, source.clone())
|
||||
@@ -7733,6 +7795,103 @@ mod tests {
|
||||
.expect_err("suspended delete must remove the requested UUID version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_retains_lifecycle_owner_through_source_cleanup() {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create decommission lifecycle-owner store");
|
||||
let (_ctx, store, shutdown) = Box::pin(without_storage_class_env(build_isolated_test_store(
|
||||
temp_dir.path(),
|
||||
"decommission-lifecycle-owner",
|
||||
&[4, 4],
|
||||
)))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = format!("decom-lifecycle-owner-{}", uuid::Uuid::new_v4());
|
||||
let object = "source.bin";
|
||||
let payload = vec![b'o'; 1_300_000];
|
||||
Box::pin(store.make_bucket(&bucket, &MakeBucketOptions::default()))
|
||||
.await
|
||||
.expect("create ordinary source bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("source bucket incarnation");
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
Box::pin(seed_decommission_source(&store, &bucket, object, payload.clone(), &opts)).await;
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let expected = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("load the exact cleanup snapshot")
|
||||
.expect("source object must exist");
|
||||
let source_disks = source_set.disk_inventory().await;
|
||||
assert_eq!(source_disks.iter().flatten().count(), 4, "all source disks must be online");
|
||||
let owner = store
|
||||
.acquire_bucket_incarnation_fence(&bucket, incarnation)
|
||||
.await
|
||||
.expect("hold source bucket lifecycle");
|
||||
let rejected = Box::pin(crate::data_movement::cleanup_source_entry_if_unchanged(
|
||||
Arc::clone(&source_set),
|
||||
&bucket,
|
||||
object,
|
||||
&expected,
|
||||
&[],
|
||||
&[],
|
||||
crate::data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: Some(incarnation),
|
||||
lifecycle_guard: owner.namespace_lock_guard(),
|
||||
..Default::default()
|
||||
},
|
||||
"test_unowned_source_cleanup",
|
||||
))
|
||||
.await
|
||||
.expect_err("a bare lifecycle signal must not authorize source cleanup");
|
||||
assert!(
|
||||
rejected
|
||||
.to_string()
|
||||
.contains("write requires an owned bucket lifecycle fence"),
|
||||
"cleanup must reject the missing owner: {rejected}"
|
||||
);
|
||||
let mut source = source_set
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("rejected cleanup must preserve the readable source");
|
||||
let mut remaining = Vec::new();
|
||||
source
|
||||
.stream
|
||||
.read_to_end(&mut remaining)
|
||||
.await
|
||||
.expect("drain preserved source");
|
||||
assert_eq!(remaining, payload);
|
||||
drop(source);
|
||||
drop(owner);
|
||||
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
Box::pin(run_decommission_entry_retry_test(
|
||||
&store,
|
||||
CancellationToken::new(),
|
||||
&bucket,
|
||||
object,
|
||||
Some(incarnation),
|
||||
Arc::new(AtomicUsize::new(0)),
|
||||
))
|
||||
.await
|
||||
.expect("decommission must copy and clean the ordinary bucket entry with its real owner");
|
||||
assert_eq!(read_decommission_target_body(&store, &bucket, object, &opts).await, payload);
|
||||
assert_decommission_source_absent(&store, &bucket, object, &opts).await;
|
||||
for disk in source_disks.into_iter().flatten() {
|
||||
assert!(
|
||||
!tokio::fs::try_exists(disk.path().join(&bucket).join(object))
|
||||
.await
|
||||
.expect("inspect the physical source path"),
|
||||
"source metadata and shards must be removed from every disk"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() {
|
||||
@@ -21073,10 +21232,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
|
||||
Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
|
||||
fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
|
||||
run_large_stack_async_test(
|
||||
"legacy-recovery-disposition",
|
||||
legacy_recovery_disposition_removes_only_local_journals_and_replays_case,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -0,0 +1,757 @@
|
||||
use super::{build_isolated_test_store, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{
|
||||
metadata::{
|
||||
BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, BucketMetadata,
|
||||
load_bucket_metadata_parse_with_presence,
|
||||
},
|
||||
metadata_sys,
|
||||
versioning::VersioningApi,
|
||||
},
|
||||
disk::{BUCKET_META_PREFIX, DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
runtime::instance::InstanceContext,
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
namespace::NamespaceLocking as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PREFIX: &str = "outputs/";
|
||||
const ENABLED: &[u8] = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";
|
||||
const SUSPENDED: &[u8] = b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>";
|
||||
const EXCLUDED_PREFIX: &[u8] = b"<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>outputs/</Prefix></ExcludedPrefixes></VersioningConfiguration>";
|
||||
const EXCLUDED_FOLDERS: &[u8] =
|
||||
b"<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>";
|
||||
const TAGGING: &[u8] = b"<Tagging><TagSet><Tag><Key>purpose</Key><Value>artifact-test</Value></Tag></TagSet></Tagging>";
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-bucket", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn drain_before_reopen(store: &ECStore, shutdown: &CancellationToken, bucket: &str) {
|
||||
shutdown.cancel();
|
||||
assert!(
|
||||
store.ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
|
||||
"ARTIFACT_PREVIOUS_STORE_MUTATIONS_DRAINED"
|
||||
);
|
||||
// Internal metadata PUTs can acknowledge quorum before their disk tails finish.
|
||||
// Acquire each key separately to wait for its actual writer to release the lock.
|
||||
for file in [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE] {
|
||||
let key = format!("{BUCKET_META_PREFIX}/{bucket}/{file}");
|
||||
let namespace = store.pools[0].disk_set[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &key)
|
||||
.await
|
||||
.expect("resolve persisted identity namespace");
|
||||
drop(
|
||||
namespace
|
||||
.get_write_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("identity write tail drained"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn dedicated_bucket(store: &ECStore, bucket: &str) {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some(PREFIX.to_owned()),
|
||||
// Creation must enforce versioning without relying on the request flag.
|
||||
versioning_enabled: false,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a dedicated artifact bucket on real disks");
|
||||
}
|
||||
|
||||
async fn persisted(store: &Arc<ECStore>, bucket: &str) -> BucketMetadata {
|
||||
let (metadata, exists) = load_bucket_metadata_parse_with_presence(store.clone(), bucket, true)
|
||||
.await
|
||||
.expect("read actual persisted bucket metadata");
|
||||
assert!(exists, "ARTIFACT_METADATA_PERSISTED");
|
||||
metadata
|
||||
}
|
||||
|
||||
fn assert_namespace(metadata: &BucketMetadata, incarnation: Uuid) {
|
||||
assert_eq!(metadata.artifact_base_prefix.as_deref(), Some(PREFIX), "ARTIFACT_NAMESPACE_IMMUTABLE");
|
||||
assert_eq!(metadata.bucket_incarnation_id, incarnation, "ARTIFACT_INCARNATION_IMMUTABLE");
|
||||
assert!(
|
||||
metadata.versioning_config.as_ref().is_some_and(VersioningApi::enabled),
|
||||
"ARTIFACT_VERSIONING_REQUIRED"
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_persisted_and_cached(store: &Arc<ECStore>, bucket: &str, incarnation: Uuid) {
|
||||
assert_namespace(&persisted(store, bucket).await, incarnation);
|
||||
let cached = metadata_sys::get_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read cached bucket identity");
|
||||
assert_namespace(&cached, incarnation);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_creation_enables_versioning_and_survives_store_reopen() {
|
||||
let root = tempfile::tempdir().expect("artifact bucket store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-created-reopen";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let created = persisted(&store, bucket).await;
|
||||
let incarnation = created.bucket_incarnation_id;
|
||||
assert!(!incarnation.is_nil(), "ARTIFACT_INCARNATION_REQUIRED");
|
||||
assert!(created.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
assert_namespace(&created, incarnation);
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
metadata_sys::reload_bucket_metadata(store.clone(), bucket)
|
||||
.await
|
||||
.expect("load the reopened bucket from persisted metadata");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary force-create replay preserves the dedicated namespace");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_ordinary_configuration_update_preserves_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact configuration store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-config-update";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let incarnation = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
metadata_sys::update_in(&store.ctx, bucket, BUCKET_TAGGING_CONFIG, TAGGING.to_vec())
|
||||
.await
|
||||
.expect("ordinary tagging update remains supported");
|
||||
let changed = persisted(&store, bucket).await;
|
||||
assert_eq!(changed.tagging_config_xml, TAGGING, "ARTIFACT_ORDINARY_CONFIG_UPDATED");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let suspended = crate::bucket::utils::deserialize(SUSPENDED).expect("valid suspended versioning XML");
|
||||
for stale_typed_config in [None, Some(suspended)] {
|
||||
let mut replacement = changed.clone();
|
||||
replacement.versioning_config = stale_typed_config;
|
||||
sys.set_validated(replacement)
|
||||
.await
|
||||
.expect("validated cache setter reparses authoritative XML");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_create_rejects_existing_bucket_type_and_prefix_changes() {
|
||||
let root = tempfile::tempdir().expect("artifact immutable creation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let ordinary = "artifact-existing-ordinary";
|
||||
store
|
||||
.make_bucket(ordinary, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create ordinary bucket");
|
||||
let original = persisted(&store, ordinary).await;
|
||||
assert!(original.artifact_base_prefix.is_none());
|
||||
for force_create in [false, true] {
|
||||
let result = store
|
||||
.make_bucket(
|
||||
ordinary,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some(PREFIX.to_owned()),
|
||||
force_create,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: ordinary bucket cannot change type");
|
||||
let unchanged = persisted(&store, ordinary).await;
|
||||
assert_eq!(unchanged.artifact_base_prefix, None);
|
||||
assert_eq!(unchanged.bucket_incarnation_id, original.bucket_incarnation_id);
|
||||
assert_eq!(unchanged.versioning_config_xml, original.versioning_config_xml);
|
||||
}
|
||||
let dedicated = "artifact-existing-dedicated";
|
||||
dedicated_bucket(&store, dedicated).await;
|
||||
let incarnation = persisted(&store, dedicated).await.bucket_incarnation_id;
|
||||
for force_create in [false, true] {
|
||||
let result = store
|
||||
.make_bucket(
|
||||
dedicated,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("another/".to_owned()),
|
||||
force_create,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: create cannot replace an existing prefix");
|
||||
assert_persisted_and_cached(&store, dedicated, incarnation).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_full_metadata_replacement_cannot_change_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact metadata replacement store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let dedicated = "artifact-replacement-dedicated";
|
||||
let ordinary = "artifact-replacement-ordinary";
|
||||
dedicated_bucket(&store, dedicated).await;
|
||||
store
|
||||
.make_bucket(ordinary, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create ordinary replacement fixture");
|
||||
let original = persisted(&store, dedicated).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for route in 0..3 {
|
||||
for prefix in [None, Some("replacement/".to_owned())] {
|
||||
let mut replacement = original.clone();
|
||||
replacement.artifact_base_prefix = prefix;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_NAMESPACE_IMMUTABLE: complete replacement cannot remove or change prefix"
|
||||
);
|
||||
assert_persisted_and_cached(&store, dedicated, incarnation).await;
|
||||
}
|
||||
let mut replacement = persisted(&store, ordinary).await;
|
||||
let ordinary_incarnation = replacement.bucket_incarnation_id;
|
||||
replacement.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
replacement.versioning_config_xml = ENABLED.to_vec();
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_NAMESPACE_IMMUTABLE: complete replacement cannot promote ordinary bucket"
|
||||
);
|
||||
let unchanged = persisted(&store, ordinary).await;
|
||||
assert!(unchanged.artifact_base_prefix.is_none());
|
||||
assert_eq!(unchanged.bucket_incarnation_id, ordinary_incarnation);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_versioning_cannot_be_suspended_or_removed() {
|
||||
let root = tempfile::tempdir().expect("artifact mandatory versioning store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-required-versioning";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let original = persisted(&store, bucket).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for xml in [SUSPENDED, b"".as_slice(), EXCLUDED_PREFIX, EXCLUDED_FOLDERS] {
|
||||
let result = metadata_sys::update_in(&store.ctx, bucket, BUCKET_VERSIONING_CONFIG, xml.to_vec()).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED: configuration update cannot disable versioning"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
for route in 0..3 {
|
||||
let mut replacement = original.clone();
|
||||
replacement.versioning_config_xml = xml.to_vec();
|
||||
replacement.versioning_config = None;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED: complete replacement cannot disable versioning"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_pending_creation_reopen_retry_preserves_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact pending creation store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-pending-retry";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(bucket);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
let incarnation = intent.bucket_incarnation_id;
|
||||
// Persist through the production intent writer, before any physical bucket exists.
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("persist real creation intent");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert!(!pending.bucket_creation_committed, "ARTIFACT_CREATION_PENDING");
|
||||
assert_namespace(&pending, incarnation);
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists(),
|
||||
"ARTIFACT_PHYSICAL_BUCKET_NOT_CREATED"
|
||||
);
|
||||
}
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let result = store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("replacement/".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: pending intent rejects a changed request");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert_namespace(&pending, incarnation);
|
||||
assert!(!pending.bucket_creation_committed);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("retry resumes persisted creation identity");
|
||||
let completed = persisted(&store, bucket).await;
|
||||
assert!(completed.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).is_dir(),
|
||||
"ARTIFACT_PHYSICAL_BUCKET_CREATED"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_partial_volume_retry_preserves_pending_identity() {
|
||||
let root = tempfile::tempdir().expect("artifact partial volume store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-partial-volume";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(bucket);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
let incarnation = intent.bucket_incarnation_id;
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("persist real creation intent before a partial physical creation");
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("first real disk")
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("create only one physical volume");
|
||||
assert!(root.path().join(format!("pool0/set0/disk0/{bucket}")).is_dir());
|
||||
for disk in 1..4 {
|
||||
assert!(!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists());
|
||||
}
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect_err("partial physical creation requires explicit recovery");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert_namespace(&pending, incarnation);
|
||||
assert!(!pending.bucket_creation_committed, "ARTIFACT_PARTIAL_CREATION_STAYS_PENDING");
|
||||
for disk in 1..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists(),
|
||||
"ARTIFACT_PARTIAL_CREATION_NOT_SILENTLY_COMPLETED"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_metadata_replacement_cannot_spoof_incarnation() {
|
||||
let root = tempfile::tempdir().expect("artifact spoofed incarnation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-spoofed-incarnation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let original = persisted(&store, bucket).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for route in 0..4 {
|
||||
for spoofed in [Uuid::nil(), Uuid::new_v4()] {
|
||||
for (prefix, xml) in [
|
||||
(None, b"".as_slice()),
|
||||
(None, SUSPENDED),
|
||||
(Some("replacement/"), ENABLED),
|
||||
(Some(PREFIX), ENABLED),
|
||||
] {
|
||||
let mut replacement = original.clone();
|
||||
replacement.bucket_incarnation_id = spoofed;
|
||||
replacement.artifact_base_prefix = prefix.map(str::to_owned);
|
||||
replacement.versioning_config_xml = xml.to_vec();
|
||||
replacement.versioning_config = None;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
2 => sys.set_validated(replacement).await,
|
||||
_ => replacement.save_with_store(store.clone()).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_INCARNATION_IMMUTABLE: route={route}, spoofed={spoofed}, prefix={prefix:?}"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_real_delete_recreate_allows_new_namespace_and_incarnation() {
|
||||
let root = tempfile::tempdir().expect("artifact genuine recreation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-genuine-recreation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let mut incarnation = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
for prefix in [Some("replacement/"), None] {
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete the empty bucket through production storage");
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: prefix.map(str::to_owned),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("genuine recreation may choose a new bucket namespace");
|
||||
let recreated = persisted(&store, bucket).await;
|
||||
assert!(!recreated.bucket_incarnation_id.is_nil(), "ARTIFACT_INCARNATION_REQUIRED");
|
||||
assert_ne!(recreated.bucket_incarnation_id, incarnation, "ARTIFACT_RECREATION_NEW_INCARNATION");
|
||||
assert_eq!(recreated.artifact_base_prefix.as_deref(), prefix, "ARTIFACT_RECREATION_NEW_NAMESPACE");
|
||||
assert_eq!(recreated.versioning_config.as_ref().is_some_and(VersioningApi::enabled), prefix.is_some());
|
||||
assert!(recreated.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
let cached = metadata_sys::get_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("recreated identity cached");
|
||||
assert_eq!(cached.bucket_incarnation_id, recreated.bucket_incarnation_id);
|
||||
assert_eq!(cached.artifact_base_prefix.as_deref(), prefix);
|
||||
incarnation = recreated.bucket_incarnation_id;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn artifact_bucket_create_observes_current_generation() {
|
||||
let root = tempfile::tempdir().expect("creation observation storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-create-observation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let initial = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete before observation");
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let current = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
assert_ne!(initial, current);
|
||||
// Invoke the exact observation phase consumed by make_artifact_bucket,
|
||||
// after real replacement between its creation and observation phases.
|
||||
assert_eq!(
|
||||
store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("observe replacement"),
|
||||
current,
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION"
|
||||
);
|
||||
assert!(
|
||||
store.observe_artifact_bucket_incarnation(bucket, "other/").await.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: mismatched namespace cannot produce a receipt"
|
||||
);
|
||||
let attempt = format!("{PREFIX}61/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, initial, &attempt)
|
||||
.await
|
||||
.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: old incarnation cannot seal same prefix"
|
||||
);
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, current, &attempt)
|
||||
.await
|
||||
.expect("current identity seals");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, reopened, stop) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
reopened
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("reopen observed identity"),
|
||||
current,
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION"
|
||||
);
|
||||
stop.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn artifact_bucket_create_observation_rejects_corrupt_authority() {
|
||||
use crate::{
|
||||
config::com::{read_config, save_config_with_opts},
|
||||
object_api::{ObjectOptions, WriteCompletion},
|
||||
};
|
||||
let root = tempfile::tempdir().expect("creation observation faults");
|
||||
let (mut ctx, mut store, mut shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-create-corruption";
|
||||
let incarnation = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("actual create and observation");
|
||||
let original = persisted(&store, bucket).await;
|
||||
let sys = ctx.bucket_metadata_sys().expect("actual metadata authority");
|
||||
// These faults target the authoritative cache branch actually read by the
|
||||
// observation. Public metadata updates reject them before publication.
|
||||
for fault in 0..6 {
|
||||
let mut damaged = original.clone();
|
||||
match fault {
|
||||
0 => damaged.name = "wrong-bucket".into(),
|
||||
1 => damaged.artifact_base_prefix = Some("other/".into()),
|
||||
2 => damaged.versioning_config = None,
|
||||
3 => damaged.versioning_config.as_mut().expect("versioning").exclude_folders = Some(true),
|
||||
4 => damaged.versioning_config = Some(crate::bucket::utils::deserialize(SUSPENDED).expect("suspended XML")),
|
||||
_ => {
|
||||
damaged.versioning_config = Some(crate::bucket::utils::deserialize(EXCLUDED_PREFIX).expect("excluded prefix XML"))
|
||||
}
|
||||
}
|
||||
sys.read().await.set_for_test(bucket.into(), Arc::new(damaged)).await;
|
||||
assert!(
|
||||
store.observe_artifact_bucket_incarnation(bucket, PREFIX).await.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: corrupt cached authority fault={fault}"
|
||||
);
|
||||
sys.read().await.set_for_test(bucket.into(), Arc::new(original.clone())).await;
|
||||
}
|
||||
drop(sys);
|
||||
for (file, corrupt, expected_error) in [
|
||||
(BUCKET_METADATA_FILE, b"corrupt metadata".to_vec(), "read_bucket_metadata: format invalid"),
|
||||
(
|
||||
BUCKET_INCARNATION_FILE,
|
||||
Uuid::nil().as_bytes().to_vec(),
|
||||
"persisted bucket incarnation is nil",
|
||||
),
|
||||
] {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{bucket}/{file}");
|
||||
let saved = read_config(store.clone(), &path).await.expect("real persisted record");
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
save_config_with_opts(store.clone(), &path, corrupt, &opts)
|
||||
.await
|
||||
.expect("inject physical record fault");
|
||||
// Reopen with a fresh metadata authority so observation must read the
|
||||
// actual damaged record, without a cached value or a negative entry.
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
(ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let error = store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("physical corruption cannot produce a receipt");
|
||||
assert!(
|
||||
error.to_string().contains(expected_error),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: corrupt physical authority {file}: {error}"
|
||||
);
|
||||
save_config_with_opts(store.clone(), &path, saved, &opts)
|
||||
.await
|
||||
.expect("restore physical record");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
(ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("restored observation"),
|
||||
incarnation
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_create_requires_namespace_locking_before_creation() {
|
||||
temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
|
||||
let root = tempfile::tempdir().expect("disabled lock storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert!(ctx.lock_manager().is_disabled());
|
||||
let bucket = "artifact-create-no-lock";
|
||||
let error = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("creation requires real locks");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket creation requires namespace locking")
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"ARTIFACT_CREATE_LOCK_REQUIRED: rejection precedes physical creation"
|
||||
);
|
||||
shutdown.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_inspect_reopens_without_migrating_other_buckets() {
|
||||
use crate::config::com::read_config;
|
||||
let root = tempfile::tempdir().expect("inspect storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-inspect-reopen";
|
||||
let incarnation = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("managed bucket");
|
||||
let pending = "artifact-inspect-pending";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(pending);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("real pending intent");
|
||||
// Physical quorum exists: a migrating reader could silently complete this intent.
|
||||
let legacy = "artifact-inspect-legacy";
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await.into_iter().flatten() {
|
||||
for name in [legacy, pending] {
|
||||
disk.make_volume(name).await.expect("real preexisting volume");
|
||||
}
|
||||
}
|
||||
let pending_path = format!("{BUCKET_META_PREFIX}/{pending}/{BUCKET_METADATA_FILE}");
|
||||
let before = read_config(store.clone(), &pending_path).await.expect("pending bytes");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drain_before_reopen(&store, &shutdown, pending).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.inspect_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("fresh-instance inspection"),
|
||||
incarnation
|
||||
);
|
||||
for name in [legacy, pending, "artifact-inspect-absent"] {
|
||||
assert!(store.inspect_artifact_bucket(name, PREFIX).await.is_err(), "ARTIFACT_INSPECT_READ_ONLY");
|
||||
}
|
||||
assert_eq!(
|
||||
read_config(store.clone(), &pending_path).await.expect("pending remains"),
|
||||
before,
|
||||
"ARTIFACT_INSPECT_READ_ONLY"
|
||||
);
|
||||
assert!(!persisted(&store, pending).await.bucket_creation_committed);
|
||||
for name in [legacy, "artifact-inspect-absent"] {
|
||||
for file in [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE] {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{name}/{file}");
|
||||
assert!(
|
||||
read_config(store.clone(), &path).await.is_err(),
|
||||
"ARTIFACT_INSPECT_READ_ONLY: no migration record"
|
||||
);
|
||||
}
|
||||
}
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
!root
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk}/artifact-inspect-absent"))
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
assert!(store.inspect_artifact_bucket(bucket, "wrong/").await.is_err());
|
||||
assert_eq!(store.inspect_artifact_bucket(bucket, PREFIX).await.unwrap(), incarnation);
|
||||
store
|
||||
.ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
|
||||
.await;
|
||||
let error = store
|
||||
.inspect_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("file endpoints cannot use distributed lock mode");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact PUT sealing requires local storage or a distributed Unix URL topology")
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_inspect_requires_namespace_locking() {
|
||||
temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
|
||||
let root = tempfile::tempdir().expect("disabled inspect locking");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert!(ctx.lock_manager().is_disabled());
|
||||
let error = store
|
||||
.inspect_artifact_bucket("artifact-inspect-no-lock", PREFIX)
|
||||
.await
|
||||
.expect_err("locking required before reading");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket inspection requires namespace locking")
|
||||
);
|
||||
shutdown.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::metadata_sys,
|
||||
disk::{Disk, local::artifact_inventory::ArtifactInventoryFrame},
|
||||
object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_managed_put_zero_and_versions_are_complete() {
|
||||
assert_managed_put_inventory_is_complete(ShardIntegrityWriteMode::Legacy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_protected_managed_put_versions_are_complete() {
|
||||
assert_managed_put_inventory_is_complete(ShardIntegrityWriteMode::Protected);
|
||||
}
|
||||
|
||||
fn assert_managed_put_inventory_is_complete(mode: ShardIntegrityWriteMode) {
|
||||
run_large_stack_async_test("artifact-inventory-managed", move || async move {
|
||||
let root = tempfile::tempdir().expect("managed inventory disk roots");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "inventory-managed", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "artifact-inventory-managed";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("actual managed versioned bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("actual incarnation");
|
||||
let prefix = format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
let keys = vec![
|
||||
(format!("{prefix}zero"), Vec::new()),
|
||||
(format!("{prefix}inline"), vec![0xA5; 1024]),
|
||||
(format!("{prefix}payload"), vec![0xA5; 1024 * 1024]),
|
||||
];
|
||||
let mut expected = std::collections::BTreeSet::new();
|
||||
for (key, data) in &keys {
|
||||
let object = store
|
||||
.put_object(
|
||||
bucket,
|
||||
key,
|
||||
&mut PutObjReader::from_vec(data.clone()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("real managed PUT including empty, inline and sharded payloads");
|
||||
expected.insert((key.clone(), object.version_id.expect("exact managed version")));
|
||||
}
|
||||
let identity = store
|
||||
.artifact_receiver_identity(bucket, incarnation, &prefix, true)
|
||||
.await
|
||||
.expect("metadata-derived identity");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_writes(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("actual all-writes seal");
|
||||
for member in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let member = member.expect("configured local member");
|
||||
let Disk::Local(local) = member.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let local = local.get_disk();
|
||||
let id = local.artifact_disk_id().await.expect("fresh actual disk identity");
|
||||
let (output, mut input) = tokio::sync::mpsc::channel(2);
|
||||
let (result, frames) = tokio::join!(
|
||||
local.inventory_artifact_attempt(
|
||||
id,
|
||||
bucket,
|
||||
&prefix,
|
||||
identity,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output
|
||||
),
|
||||
async {
|
||||
let mut frames = Vec::new();
|
||||
while let Some(frame) = input.recv().await {
|
||||
frames.push(frame);
|
||||
}
|
||||
frames
|
||||
}
|
||||
);
|
||||
assert!(result.is_ok(), "A03I1_REAL_MANAGED_PUT_SCAN: {result:?}");
|
||||
let actual = frames
|
||||
.iter()
|
||||
.filter_map(|frame| match frame {
|
||||
ArtifactInventoryFrame::Entries { versions, .. } => Some(versions),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.map(|version| (version.key.clone(), version.version_id))
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(actual, expected, "A03I1_REAL_ZERO_INLINE_AND_SHARDED_VERSIONS");
|
||||
assert!(matches!(frames.last(), Some(ArtifactInventoryFrame::Complete { entries: 3, .. })));
|
||||
}
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
async fn inventory_fixture() -> (
|
||||
tempfile::TempDir,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
tokio_util::sync::CancellationToken,
|
||||
Uuid,
|
||||
String,
|
||||
) {
|
||||
let root = tempfile::tempdir().expect("inventory aggregate roots");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "inventory-aggregate", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(
|
||||
"inventory-aggregate",
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".into()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("managed bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id("inventory-aggregate")
|
||||
.await
|
||||
.expect("actual incarnation");
|
||||
let prefix = format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
store
|
||||
.put_object(
|
||||
"inventory-aggregate",
|
||||
&format!("{prefix}object"),
|
||||
&mut PutObjReader::from_vec(vec![1; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("actual managed object");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_writes("inventory-aggregate", incarnation, &prefix)
|
||||
.await
|
||||
.expect("actual two-domain seal");
|
||||
(root, store, shutdown, incarnation, prefix)
|
||||
}
|
||||
|
||||
async fn aggregate(
|
||||
store: std::sync::Arc<crate::store::ECStore>,
|
||||
incarnation: Uuid,
|
||||
prefix: &str,
|
||||
) -> (
|
||||
crate::error::Result<()>,
|
||||
Vec<crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame>,
|
||||
) {
|
||||
let (output, mut receiver) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async {
|
||||
let mut frames = Vec::new();
|
||||
while let Some(frame) = receiver.recv().await {
|
||||
frames.push(frame);
|
||||
}
|
||||
frames
|
||||
};
|
||||
tokio::join!(
|
||||
store.inventory_artifact_attempt(
|
||||
"inventory-aggregate",
|
||||
incarnation,
|
||||
prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output
|
||||
),
|
||||
drain
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_all_members_required() {
|
||||
run_large_stack_async_test("artifact-inventory-all-members", || async {
|
||||
use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let (result, frames) = aggregate(store.clone(), incarnation, &prefix).await;
|
||||
result.expect("all four real sealed disks scanned");
|
||||
let observed: std::collections::BTreeSet<_> = frames
|
||||
.iter()
|
||||
.filter_map(|frame| match frame {
|
||||
ArtifactAttemptInventoryFrame::Observations {
|
||||
endpoint,
|
||||
disk_id,
|
||||
versions,
|
||||
} => {
|
||||
assert_eq!(versions.len(), 1);
|
||||
assert_eq!(versions[0].key, format!("{prefix}object"));
|
||||
Some((endpoint.clone(), *disk_id))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(observed.len(), 4, "A03I2_ALL_MEMBERS");
|
||||
assert!(
|
||||
matches!(
|
||||
frames.last(),
|
||||
Some(ArtifactAttemptInventoryFrame::Complete {
|
||||
members: 4,
|
||||
observations: 4
|
||||
})
|
||||
),
|
||||
"A03I2_ALL_MEMBERS"
|
||||
);
|
||||
let original = store.pools[0].disk_set[0].disks.write().await[3].take();
|
||||
let (denied, frames) = aggregate(store.clone(), incarnation, &prefix).await;
|
||||
assert!(denied.is_err(), "one missing configured member is incomplete");
|
||||
assert!(
|
||||
!frames
|
||||
.iter()
|
||||
.any(|frame| matches!(frame, ArtifactAttemptInventoryFrame::Complete { .. }))
|
||||
);
|
||||
store.pools[0].disk_set[0].disks.write().await[3] = original;
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_membership_change_is_incomplete() {
|
||||
run_large_stack_async_test("artifact-inventory-membership", || async {
|
||||
use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let original = store.pools[0].disk_set[0].disk_inventory().await[0]
|
||||
.clone()
|
||||
.expect("first member");
|
||||
let Disk::Local(local) = original.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let replacement = std::sync::Arc::new(Disk::Local(local.clone()));
|
||||
let (output, mut receiver) = tokio::sync::mpsc::channel(1);
|
||||
let owned = store.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
owned
|
||||
.inventory_artifact_attempt(
|
||||
"inventory-aggregate",
|
||||
incarnation,
|
||||
&prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output,
|
||||
)
|
||||
.await
|
||||
});
|
||||
assert!(matches!(receiver.recv().await, Some(ArtifactAttemptInventoryFrame::Observations { .. })));
|
||||
store.pools[0].disk_set[0].disks.write().await[0] = Some(replacement);
|
||||
let mut complete = false;
|
||||
while let Some(frame) = receiver.recv().await {
|
||||
complete |= matches!(frame, ArtifactAttemptInventoryFrame::Complete { .. });
|
||||
}
|
||||
let error = worker
|
||||
.await
|
||||
.expect("aggregate worker")
|
||||
.expect_err("same endpoint/UUID but changed captured Arc is incomplete");
|
||||
assert!(error.to_string().contains("membership changed"), "{error}");
|
||||
assert!(!complete);
|
||||
store.pools[0].disk_set[0].disks.write().await[0] = Some(original);
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_readonly_authority_cold_cache_and_queued_writer() {
|
||||
run_large_stack_async_test("artifact-inventory-readonly-authority", || async {
|
||||
use crate::{
|
||||
disk::{BUCKET_META_PREFIX, DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
storage_api_contracts::namespace::NamespaceLocking as _,
|
||||
};
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let member = store.pools[0].disk_set[0].disk_inventory().await[0]
|
||||
.clone()
|
||||
.expect("real receiver");
|
||||
let Disk::Local(local) = member.as_ref() else {
|
||||
panic!("local");
|
||||
};
|
||||
let expected = local.artifact_disk_id().await.expect("actual disk");
|
||||
let _lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock("inventory-aggregate")
|
||||
.await
|
||||
.expect("coordinator lifecycle owner");
|
||||
metadata_sys::remove_bucket_metadata_in(&store.ctx, "inventory-aggregate")
|
||||
.await
|
||||
.expect("cold metadata cache");
|
||||
let lock = store
|
||||
.new_ns_lock("inventory-aggregate", "inventory-aggregate")
|
||||
.await
|
||||
.expect("business bucket namespace");
|
||||
let reader = lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("existing namespace reader");
|
||||
let mut writer = Box::pin(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()));
|
||||
assert!(
|
||||
matches!(futures::poll!(writer.as_mut()), std::task::Poll::Pending),
|
||||
"real queued business bucket WRITE"
|
||||
);
|
||||
let (output, mut frames) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async { while frames.recv().await.is_some() {} };
|
||||
let endpoint = member.endpoint().to_string();
|
||||
let receive = store.inventory_local_artifact_attempt(
|
||||
(&endpoint, expected),
|
||||
("inventory-aggregate", incarnation),
|
||||
&prefix,
|
||||
rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase::ArtifactInventoryScan,
|
||||
tokio::time::Instant::now() + Duration::from_secs(2),
|
||||
output,
|
||||
);
|
||||
let (result, ()) = tokio::join!(receive, drain);
|
||||
result.expect("A03I2_READONLY_AUTHORITY_NO_REENTRY");
|
||||
drop(writer);
|
||||
drop(reader);
|
||||
drop(_lifecycle);
|
||||
// A legacy physical bucket lacking both authoritative records must
|
||||
// stay legacy after rejected inventory; no implicit getter migration.
|
||||
store
|
||||
.make_bucket(
|
||||
"inventory-legacy",
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary legacy fixture");
|
||||
let members = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
for member in members.iter().flatten() {
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
let path = member
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join("inventory-legacy")
|
||||
.join(file);
|
||||
if path.exists() {
|
||||
std::fs::remove_dir_all(&path).expect("remove fixture-only creation facts");
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata_sys::remove_bucket_metadata_in(&store.ctx, "inventory-legacy")
|
||||
.await
|
||||
.expect("cold legacy cache");
|
||||
let (output, mut frames) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async { while frames.recv().await.is_some() {} };
|
||||
let (denied, ()) = tokio::join!(
|
||||
store.clone().inventory_artifact_attempt(
|
||||
"inventory-legacy",
|
||||
Uuid::new_v4(),
|
||||
&prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(2),
|
||||
output
|
||||
),
|
||||
drain
|
||||
);
|
||||
assert!(denied.is_err());
|
||||
for member in members.iter().flatten() {
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
assert!(
|
||||
!member
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join("inventory-legacy")
|
||||
.join(file)
|
||||
.exists(),
|
||||
"A03I2_READONLY_NO_MIGRATION"
|
||||
);
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,587 @@
|
||||
use super::{build_isolated_test_store, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{artifact::artifact_put_seal_record_path, metadata_sys},
|
||||
config::com,
|
||||
disk::{DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
object_api::{ObjectInfo, ObjectOptions, PutObjReader, WriteCompletion},
|
||||
runtime::instance::InstanceContext,
|
||||
set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, rename_fanout_barrier},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEADLINE: Duration = Duration::from_secs(30);
|
||||
|
||||
fn attempt_prefix() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-put-seal", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn create_bucket(store: &ECStore, bucket: &str) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a real versioned artifact bucket");
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("read actual bucket incarnation")
|
||||
}
|
||||
|
||||
async fn put(store: &ECStore, bucket: &str, object: &str) -> ObjectInfo {
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0xA5; 273]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("unsealed PUT must commit on all four disks")
|
||||
}
|
||||
|
||||
async fn disk_metadata(store: &ECStore, bucket: &str, object: &str) -> Vec<Vec<u8>> {
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
assert_eq!(disks.len(), 4);
|
||||
let mut metadata = Vec::new();
|
||||
for disk in disks {
|
||||
metadata.push(
|
||||
disk.expect("real fixture disk is online")
|
||||
.read_xl(bucket, object, false)
|
||||
.await
|
||||
.expect("read actual per-disk versions")
|
||||
.buf,
|
||||
);
|
||||
}
|
||||
metadata
|
||||
}
|
||||
|
||||
async fn assert_rejected_without_new_version(store: &ECStore, bucket: &str, object: &str, before: &[Vec<u8>]) {
|
||||
let result = store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0xBB; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_PUT_SEALED: later PUT must be rejected");
|
||||
assert_eq!(
|
||||
disk_metadata(store, bucket, object).await,
|
||||
before,
|
||||
"ARTIFACT_PUT_SEALED: no disk gains a version"
|
||||
);
|
||||
}
|
||||
|
||||
async fn tail_barrier_case(cancel_put_waiter: bool) {
|
||||
let root = tempfile::tempdir().expect("artifact tail fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-tail";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let tracker = rename_fanout_barrier::observe_tasks(&object);
|
||||
let tail = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let quorum = PutObjectCommitBarrier::install(bucket, &object, PutObjectCommitPause::AfterRenameQuorum);
|
||||
let handoff = PutObjectCommitBarrier::install(bucket, &object, PutObjectCommitPause::AfterRenameHandoff);
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let put_task = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
&put_object,
|
||||
&mut PutObjReader::from_vec(vec![0xA5; 273]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, tail.wait_until_paused())
|
||||
.await
|
||||
.expect("one real disk pauses before rename");
|
||||
quorum.wait_until_paused().await;
|
||||
if cancel_put_waiter {
|
||||
put_task.abort();
|
||||
}
|
||||
quorum.release();
|
||||
handoff.wait_until_paused().await;
|
||||
handoff.release();
|
||||
let result = tokio::time::timeout(DEADLINE, put_task)
|
||||
.await
|
||||
.expect("PUT waiter returns while its tail remains paused");
|
||||
if cancel_put_waiter {
|
||||
assert!(result.expect_err("cancelled PUT waiter").is_cancelled());
|
||||
} else {
|
||||
result
|
||||
.expect("PUT task joins")
|
||||
.expect("PUT quorum ACK succeeds before tail completion");
|
||||
}
|
||||
assert!(tracker.running() > 0, "real rename tail must still be live");
|
||||
let parallel_object = format!("{prefix}parallel.png");
|
||||
let parallel = tokio::time::timeout(DEADLINE, put(&store, bucket, ¶llel_object))
|
||||
.await
|
||||
.expect("ARTIFACT_PUT_SHARED_ADMISSION: another object in the same attempt completes while A is paused");
|
||||
let mut reader = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
store.get_object_reader(
|
||||
bucket,
|
||||
¶llel_object,
|
||||
None,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(parallel.version_id.expect("parallel PUT has a real version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("parallel object GET completes while A is paused")
|
||||
.expect("parallel object is readable");
|
||||
let mut bytes = Vec::new();
|
||||
tokio::time::timeout(DEADLINE, reader.stream.read_to_end(&mut bytes))
|
||||
.await
|
||||
.expect("parallel object body completes while A is paused")
|
||||
.expect("read parallel object body");
|
||||
assert_eq!(bytes, vec![0xA5; 273], "ARTIFACT_PUT_SHARED_ADMISSION: B publishes exact bytes");
|
||||
drop(reader);
|
||||
assert!(tracker.running() > 0, "A's physical tail remains paused throughout B's PUT and GET");
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
let attempt_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &record)
|
||||
.await
|
||||
.expect("resolve the actual PUT and seal coordination namespace");
|
||||
assert!(
|
||||
attempt_lock.get_write_lock(Duration::from_secs(1)).await.is_err(),
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: the real commit owner retains the attempt lock until its disk tail drains"
|
||||
);
|
||||
let seal_store = store.clone();
|
||||
let seal_prefix = prefix.clone();
|
||||
let mut seal =
|
||||
tokio::spawn(async move { seal_store.seal_artifact_attempt_puts(bucket, incarnation, &seal_prefix).await });
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(200), &mut seal).await.is_err(),
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: seal cannot finish while a real rename tail is paused"
|
||||
);
|
||||
assert!(tracker.running() > 0, "tail remains paused throughout the seal attempt");
|
||||
tail.release();
|
||||
tokio::time::timeout(DEADLINE, seal)
|
||||
.await
|
||||
.expect("seal resumes after tail release")
|
||||
.expect("seal task joins")
|
||||
.expect("seal persists");
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
while tracker.running() != 0 || ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("actual detached write owners drain");
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
})
|
||||
.await;
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_waits_for_quorum_ack_rename_tail() {
|
||||
tail_barrier_case(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_waits_after_put_waiter_cancellation() {
|
||||
tail_barrier_case(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_survives_reopen_and_is_attempt_scoped() {
|
||||
let root = tempfile::tempdir().expect("artifact durable seal directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-reopen";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("persist first seal");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("same seal is idempotent");
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
put(&store, bucket, &format!("{}other.png", attempt_prefix())).await;
|
||||
shutdown.cancel();
|
||||
assert!(
|
||||
ctx.wait_for_detached_mutations(DEADLINE).await,
|
||||
"old instance mutations drain before reopen"
|
||||
);
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
metadata_sys::reload_bucket_metadata(store.clone(), bucket)
|
||||
.await
|
||||
.expect("reload dedicated bucket after restart");
|
||||
assert_eq!(store.bucket_incarnation_id(bucket).await.expect("reopened incarnation"), incarnation);
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("persisted seal remains idempotent");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_put_seal_rejects_corrupt_and_wrong_incarnation_records() {
|
||||
super::run_large_stack_async_test("artifact-put-corrupt", artifact_put_corrupt_record_case);
|
||||
}
|
||||
|
||||
async fn artifact_put_corrupt_record_case() {
|
||||
let root = tempfile::tempdir().expect("artifact invalid seal record directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-corrupt";
|
||||
let prefix = attempt_prefix();
|
||||
let old_incarnation = create_bucket(&store, bucket).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, old_incarnation, &prefix)
|
||||
.await
|
||||
.expect("produce a real previous-generation seal");
|
||||
let old_record = com::read_config(store.clone(), &artifact_put_seal_record_path(bucket, old_incarnation, &prefix))
|
||||
.await
|
||||
.expect("read actual seal bytes");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete empty old-generation bucket");
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
assert_ne!(incarnation, old_incarnation);
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, old_incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"wrong expected incarnation cannot seal current bucket"
|
||||
);
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
for invalid_record in [old_record, Vec::new(), b"not-json".to_vec()] {
|
||||
com::save_config_with_opts(
|
||||
store.clone(),
|
||||
&record,
|
||||
invalid_record,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("persist actual invalid seal record on all disks");
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"invalid seal cannot be treated as absent or idempotent success"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_does_not_change_ordinary_bucket_puts() {
|
||||
let root = tempfile::tempdir().expect("ordinary bucket seal regression directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-ordinary";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create ordinary versioned bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("ordinary incarnation");
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let first = put(&store, bucket, &object).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"ordinary bucket cannot acquire artifact seal semantics"
|
||||
);
|
||||
let second = put(&store, bucket, &object).await;
|
||||
assert_ne!(first.version_id, second.version_id, "ordinary versioned PUT creates another version");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_put_seal_lock_survives_physical_publication_timeout() {
|
||||
// Match large ECStore fixtures while retaining the current-thread clock required by pause/advance.
|
||||
std::thread::Builder::new()
|
||||
.name("artifact-put-physical-timeout".to_owned())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("physical timeout test runtime should build");
|
||||
runtime.block_on(artifact_put_physical_timeout_case());
|
||||
})
|
||||
.expect("physical timeout test thread should spawn")
|
||||
.join()
|
||||
.expect("physical timeout test thread should complete");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn artifact_put_physical_timeout_case() {
|
||||
use crate::disk::{Disk, ReadOptions, STORAGE_FORMAT_FILE, os};
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
|
||||
(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false")),
|
||||
],
|
||||
async {
|
||||
let root = tempfile::tempdir().expect("artifact physical timeout directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-physical-timeout";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
let mut hooks = Vec::new();
|
||||
let mut entered = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut destinations = Vec::new();
|
||||
for disk in disks.iter().flatten() {
|
||||
let Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("physical timeout fixture requires local disks");
|
||||
};
|
||||
// Resolve through the mount lease; Linux may use an FD-based path.
|
||||
let destination = local
|
||||
.get_disk()
|
||||
.get_object_path_for_io(bucket, &object)
|
||||
.expect("real publication path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
hooks.push(os::prepared_publication_test_hooks::install(
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
// Dropping the sender also releases the syscall during panic unwinding.
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
));
|
||||
entered.push(entered_rx);
|
||||
releases.push(release_tx);
|
||||
destinations.push(destination);
|
||||
}
|
||||
assert_eq!(entered.len(), 4, "all four real disks must be online");
|
||||
let version = Uuid::new_v4();
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
&put_object,
|
||||
&mut PutObjReader::from_vec(vec![0xCC; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.to_string()),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
signals = futures::future::join_all(entered) => assert!(signals.into_iter().all(|signal| signal.is_ok())),
|
||||
_ = &mut replacement => panic!("PUT cannot finish before its physical publishers are paused"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all physical xl.meta publications reach the blocking syscall");
|
||||
// All disk wrapper timers exist, but their physical closures stay blocked.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(DEADLINE, replacement)
|
||||
.await
|
||||
.expect("PUT returns after disk wrapper timeouts")
|
||||
.expect("PUT task joins");
|
||||
assert!(result.is_err(), "four timed-out publishers cannot satisfy write quorum");
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
let attempt_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &record)
|
||||
.await
|
||||
.expect("actual attempt coordination namespace");
|
||||
let physical_owner_kept_lock = attempt_lock.get_write_lock(Duration::from_secs(1)).await.is_err();
|
||||
|
||||
// Drain physical syscalls before the regression assertion, including a RED run.
|
||||
drop(releases);
|
||||
for (disk, destination) in disks.iter().flatten().zip(&destinations) {
|
||||
drop(
|
||||
tokio::time::timeout(DEADLINE, os::acquire_rename_data_mutation_lease(&disk.path(), bucket, destination))
|
||||
.await
|
||||
.expect("released physical publisher drains"),
|
||||
);
|
||||
let published = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("timed-out physical writer actually publishes the exact new version");
|
||||
assert_eq!(published.size, 379);
|
||||
}
|
||||
assert!(
|
||||
physical_owner_kept_lock,
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: disk workers retain the attempt lock after the PUT coordinator returns its timeout"
|
||||
);
|
||||
tokio::time::timeout(DEADLINE, store.clone().seal_artifact_attempt_puts(bucket, incarnation, &prefix))
|
||||
.await
|
||||
.expect("seal resumes after physical owners drain")
|
||||
.expect("seal persists after timeout recovery");
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
shutdown.cancel();
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_rejects_mismatched_topology_and_preserves_ordinary_local_puts() {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
|
||||
const TOPOLOGY_ERROR: &str = "artifact PUT sealing requires local storage or a distributed Unix URL topology";
|
||||
let root = tempfile::tempdir().expect("artifact distributed topology directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let bucket = "artifact-put-distributed";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let ordinary = "artifact-put-distributed-ordinary";
|
||||
store
|
||||
.make_bucket(
|
||||
ordinary,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create ordinary versioned control bucket");
|
||||
let ordinary_incarnation = store.bucket_incarnation_id(ordinary).await.expect("ordinary identity");
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
let first = put(&store, ordinary, &object).await;
|
||||
|
||||
// Keep every physical disk online: the gate must inspect configured topology.
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
assert!(ctx.is_dist_erasure().await);
|
||||
let rejected = store
|
||||
.put_object(
|
||||
bucket,
|
||||
&object,
|
||||
&mut PutObjReader::from_vec(vec![0xDD; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("file endpoints with distributed lock mode must be rejected before publication");
|
||||
assert!(rejected.to_string().contains(TOPOLOGY_ERROR), "ARTIFACT_PUT_TOPOLOGY: {rejected}");
|
||||
assert_eq!(
|
||||
disk_metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"ARTIFACT_PUT_TOPOLOGY: no disk gains a version"
|
||||
);
|
||||
let rejected = store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect_err("mismatched topology cannot claim a durable PUT seal");
|
||||
assert!(rejected.to_string().contains(TOPOLOGY_ERROR), "ARTIFACT_PUT_TOPOLOGY: {rejected}");
|
||||
// This local fixture has no distributed lock clients; ordinary PUT retains its local regression check.
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let second = put(&store, ordinary, &object).await;
|
||||
assert_ne!(
|
||||
first.version_id, second.version_id,
|
||||
"ordinary local PUT stays available after artifact rejection"
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id(ordinary)
|
||||
.await
|
||||
.expect("ordinary identity remains available"),
|
||||
ordinary_incarnation
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("artifact identity remains available"),
|
||||
incarnation
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,689 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{artifact::artifact_version_retirement_record_path, metadata_sys},
|
||||
config::com,
|
||||
disk::{DiskAPI as _, DiskStore, RUSTFS_META_BUCKET, ReadOptions, STORAGE_FORMAT_FILE, os},
|
||||
error::Error,
|
||||
object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion},
|
||||
runtime::instance::InstanceContext,
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
heal::HealOperations,
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEADLINE: Duration = Duration::from_secs(30);
|
||||
const PAYLOAD_SIZE: usize = 1024 * 1024;
|
||||
|
||||
fn attempt_prefix() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn put_options() -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn heal_options() -> HealOpts {
|
||||
HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-retirement", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn create_bucket(store: &ECStore, bucket: &str, artifact: bool) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: artifact.then(|| "outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create real four-disk versioned bucket");
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("read actual bucket incarnation")
|
||||
}
|
||||
|
||||
async fn disks(store: &ECStore) -> Vec<DiskStore> {
|
||||
let disks = store.pools[0].disk_set[0]
|
||||
.disk_inventory()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|disk| disk.expect("fixture disk remains online"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(disks.len(), 4);
|
||||
disks
|
||||
}
|
||||
|
||||
fn object_path(disk: &DiskStore, bucket: &str, object: &str) -> PathBuf {
|
||||
disk.get_object_path_for_io_if_local(bucket, &encode_dir_object(object))
|
||||
.expect("fixture uses local disks")
|
||||
.expect("resolve object through held mount")
|
||||
}
|
||||
|
||||
async fn file_bytes(path: &Path) -> Option<Vec<u8>> {
|
||||
match tokio::fs::read(path).await {
|
||||
Ok(bytes) => Some(bytes),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
|
||||
Err(error) => panic!("read actual fixture file: {error}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn metadata(store: &ECStore, bucket: &str, object: &str) -> Vec<Option<Vec<u8>>> {
|
||||
let mut result = Vec::new();
|
||||
for disk in disks(store).await {
|
||||
let path = object_path(&disk, bucket, object).join(STORAGE_FORMAT_FILE);
|
||||
result.push(file_bytes(&path).await);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn put_payload(store: &ECStore, bucket: &str, object: &str, byte: u8, legacy: bool) -> Uuid {
|
||||
let mut options = put_options();
|
||||
if legacy {
|
||||
// Exercise genuine supported minority metadata recovery, without fabricated proofs.
|
||||
options.shard_integrity_write_mode = Some(ShardIntegrityWriteMode::Legacy);
|
||||
}
|
||||
let body = if object.ends_with('/') {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![byte; PAYLOAD_SIZE]
|
||||
};
|
||||
store
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(body), &options)
|
||||
.await
|
||||
.expect("commit exact payload and version to all four disks")
|
||||
.version_id
|
||||
.expect("artifact PUT returns a non-null version")
|
||||
}
|
||||
|
||||
async fn assert_payload(store: &ECStore, bucket: &str, object: &str, version: Uuid, byte: u8) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(version.to_string()),
|
||||
..put_options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retirement leaves the exact committed version readable");
|
||||
assert_eq!(reader.object_info.version_id, Some(version), "GET preserves exact version identity");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read exact committed bytes");
|
||||
let expected = if object.ends_with('/') {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![byte; PAYLOAD_SIZE]
|
||||
};
|
||||
assert_eq!(body, expected, "retirement does not delete or replace committed bytes");
|
||||
}
|
||||
|
||||
async fn remove_shard(store: &ECStore, bucket: &str, object: &str, version: Uuid) -> (DiskStore, PathBuf) {
|
||||
let disk = disks(store).await.remove(0);
|
||||
let info = disk
|
||||
.read_version("", bucket, &encode_dir_object(object), &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("read exact version before removing a real shard");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(&disk, bucket, object)
|
||||
.join(info.data_dir.expect("non-inline object has a data directory").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&part).await.expect("remove one real payload shard");
|
||||
assert!(!part.exists());
|
||||
(disk, part)
|
||||
}
|
||||
|
||||
async fn assert_healthy_version(store: &ECStore, bucket: &str, object: &str, version: Uuid) {
|
||||
for disk in disks(store).await {
|
||||
let info = disk
|
||||
.read_version("", bucket, &encode_dir_object(object), &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("repaired disk retains exact committed metadata");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
assert_eq!(info.size, i64::try_from(PAYLOAD_SIZE).expect("payload size fits i64"));
|
||||
let checked = disk
|
||||
.check_parts(bucket, &encode_dir_object(object), &info)
|
||||
.await
|
||||
.expect("verify genuine repaired shard");
|
||||
assert!(!checked.results.is_empty());
|
||||
assert!(
|
||||
checked
|
||||
.results
|
||||
.iter()
|
||||
.all(|result| *result == crate::disk::CHECK_PART_SUCCESS),
|
||||
"unretired version's real repaired shard passes integrity verification"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_policy_error(error: &Error) {
|
||||
assert!(
|
||||
!matches!(
|
||||
error,
|
||||
Error::FileNotFound | Error::FileVersionNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _)
|
||||
),
|
||||
"retirement policy refusal must not claim physical absence: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_repair_rejected(store: &ECStore, bucket: &str, object: &str, version: &str) -> Error {
|
||||
let result = HealOperations::heal_object(store, bucket, object, version, &heal_options()).await;
|
||||
let error = match result {
|
||||
Err(error) | Ok((_, Some(error))) => error,
|
||||
Ok((item, None)) => {
|
||||
panic!("ARTIFACT_RETIREMENT_NO_RESURRECTION: retired exact version was accepted for real repair: {item:?}")
|
||||
}
|
||||
};
|
||||
assert_policy_error(&error);
|
||||
// The proof-returning API must preserve the same refusal without converting it
|
||||
// into authoritative absence that a later destructive caller could consume.
|
||||
match store.heal_object_with_proof(bucket, object, version, &heal_options()).await {
|
||||
Err(error) => assert_policy_error(&error),
|
||||
Ok(result) => {
|
||||
assert!(result.absence.is_none(), "retirement never produces a physical absence proof");
|
||||
assert_policy_error(result.error.as_ref().expect("retired repair returns a policy error"));
|
||||
}
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
async fn assert_retired_repair_rejected(store: &ECStore, bucket: &str, object: &str, version: &str) {
|
||||
let error = assert_repair_rejected(store, bucket, object, version).await;
|
||||
assert!(
|
||||
error.to_string().contains("artifact object version is retired"),
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: expected retirement refusal, received {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn seal(store: &Arc<ECStore>, bucket: &str, incarnation: Uuid, prefix: &str) {
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, prefix)
|
||||
.await
|
||||
.expect("persist the attempt PUT seal before retirement");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_requires_seal_exact_identity_and_canonical_directory() {
|
||||
run_large_stack_async_test("artifact-retirement-identity", || async {
|
||||
let root = tempfile::tempdir().expect("retirement identity directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-identity";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let directory = format!("{prefix}folder/");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
let directory_version = put_payload(&store, bucket, &directory, 0, false).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"unsealed attempt cannot retire a version"
|
||||
);
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let wrong_prefix = attempt_prefix();
|
||||
seal(&store, bucket, incarnation, &wrong_prefix).await;
|
||||
for (expected, attempt, target, selected) in [
|
||||
(Uuid::new_v4(), prefix.clone(), object.clone(), version),
|
||||
(incarnation, wrong_prefix, object.clone(), version),
|
||||
(incarnation, prefix.clone(), object.clone(), Uuid::nil()),
|
||||
(incarnation, prefix.clone(), prefix.clone(), version),
|
||||
] {
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, expected, &attempt, &target, selected)
|
||||
.await
|
||||
.is_err(),
|
||||
"retirement requires the exact incarnation, attempt, inner object, and non-null version"
|
||||
);
|
||||
}
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
let ordinary = "artifact-retirement-ordinary";
|
||||
let ordinary_incarnation = create_bucket(&store, ordinary, false).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(ordinary, ordinary_incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"retirement is limited to dedicated artifact buckets"
|
||||
);
|
||||
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &directory, directory_version)
|
||||
.await
|
||||
.expect("retire logical directory identity");
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &prefix, &directory, directory_version);
|
||||
assert_eq!(
|
||||
record,
|
||||
artifact_version_retirement_record_path(
|
||||
bucket,
|
||||
incarnation,
|
||||
&prefix,
|
||||
&encode_dir_object(&directory),
|
||||
directory_version
|
||||
)
|
||||
);
|
||||
let bytes = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("read durable directory retirement");
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &encode_dir_object(&directory), directory_version)
|
||||
.await
|
||||
.expect("encoded directory retirement retry is idempotent");
|
||||
assert_eq!(com::read_config(store.clone(), &record).await.expect("read unchanged record"), bytes);
|
||||
assert_retired_repair_rejected(&store, bucket, &directory, &directory_version.to_string()).await;
|
||||
assert_retired_repair_rejected(&store, bucket, &encode_dir_object(&directory), &directory_version.to_string()).await;
|
||||
assert_payload(&store, bucket, &directory, directory_version, 0).await;
|
||||
// Rejected identity requests must not retire the valid regular object.
|
||||
let (_, missing) = remove_shard(&store, bucket, &object, version).await;
|
||||
let (_, error) = HealOperations::heal_object(store.as_ref(), bucket, &object, &version.to_string(), &heal_options())
|
||||
.await
|
||||
.expect("unretired version remains repairable after rejected retirement");
|
||||
assert!(error.is_none(), "unretired repair failed: {error:?}");
|
||||
assert!(missing.exists(), "unretired version's real shard is restored");
|
||||
assert_healthy_version(&store, bucket, &object, version).await;
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_rejects_explicit_and_latest_repair_but_preserves_other_version_after_reopen() {
|
||||
run_large_stack_async_test("artifact-retirement-versions", || async {
|
||||
let root = tempfile::tempdir().expect("retired version fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-versions";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let retained = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
let retired = put_payload(&store, bucket, &object, 0xB6, false).await;
|
||||
assert_ne!(retained, retired);
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
let mut physical_parts = Vec::new();
|
||||
for disk in disks(&store).await {
|
||||
for version in [retained, retired] {
|
||||
let info = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("snapshot exact version before retirement");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(&disk, bucket, &object)
|
||||
.join(info.data_dir.expect("payload data directory").to_string())
|
||||
.join("part.1");
|
||||
physical_parts.push((part.clone(), tokio::fs::read(part).await.expect("snapshot original shard bytes")));
|
||||
}
|
||||
}
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, retired)
|
||||
.await
|
||||
.expect("persist exact latest-version retirement");
|
||||
assert_eq!(
|
||||
metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"retirement itself does not edit object metadata"
|
||||
);
|
||||
for (part, bytes) in physical_parts {
|
||||
assert_eq!(
|
||||
tokio::fs::read(part).await.expect("retirement leaves every physical shard"),
|
||||
bytes,
|
||||
"retirement changes neither version's physical data"
|
||||
);
|
||||
}
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
let (_, retired_part) = remove_shard(&store, bucket, &object, retired).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
for selected in [retired.to_string(), String::new()] {
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &selected).await;
|
||||
assert!(
|
||||
!retired_part.exists(),
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: retired shard must stay absent"
|
||||
);
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before, "retired repair publishes no metadata");
|
||||
}
|
||||
let (_, retained_part) = remove_shard(&store, bucket, &object, retained).await;
|
||||
let (_, error) = HealOperations::heal_object(store.as_ref(), bucket, &object, &retained.to_string(), &heal_options())
|
||||
.await
|
||||
.expect("another version on the same object remains repairable");
|
||||
assert!(error.is_none(), "other-version repair failed: {error:?}");
|
||||
assert!(retained_part.exists(), "other version's actual shard is restored");
|
||||
assert_healthy_version(&store, bucket, &object, retained).await;
|
||||
assert!(!retired_part.exists(), "other-version repair must not restore the retired shard");
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(store.bucket_incarnation_id(bucket).await.expect("reopened incarnation"), incarnation);
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &retired.to_string()).await;
|
||||
assert!(!retired_part.exists(), "reopen cannot forget exact-version retirement");
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_blocks_minority_metadata_regeneration() {
|
||||
run_large_stack_async_test("artifact-retirement-regeneration", || async {
|
||||
let root = tempfile::tempdir().expect("minority metadata fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-regeneration";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, true).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.expect("persist version retirement before minority metadata damage");
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
let mut parts = Vec::new();
|
||||
for (index, disk) in disks(&store).await.iter().enumerate() {
|
||||
let info = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("read genuine legacy version before damage");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(disk, bucket, &object)
|
||||
.join(info.data_dir.expect("legacy payload has data directory").to_string())
|
||||
.join("part.1");
|
||||
parts.push((part.clone(), tokio::fs::read(&part).await.expect("snapshot real shard bytes")));
|
||||
if index > 0 {
|
||||
tokio::fs::remove_file(object_path(disk, bucket, &object).join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("leave exactly one genuine metadata copy");
|
||||
}
|
||||
}
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
assert_eq!(before.iter().filter(|entry| entry.is_some()).count(), 1);
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert_eq!(
|
||||
metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: minority regeneration must not publish retired metadata"
|
||||
);
|
||||
for (path, bytes) in parts {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("retired shard remains physically present"), bytes);
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_corrupt_or_wrong_identity_records_fail_closed() {
|
||||
run_large_stack_async_test("artifact-retirement-records", || async {
|
||||
let root = tempfile::tempdir().expect("corrupt retirement fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-records";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.expect("create genuine retirement record");
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &prefix, &object, version);
|
||||
let original = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("read genuine retirement JSON");
|
||||
let decoded: serde_json::Value = serde_json::from_slice(&original).expect("retirement record is JSON");
|
||||
let mut invalid = vec![Vec::new(), b"not-json".to_vec()];
|
||||
for (field, value) in [
|
||||
("format_version", serde_json::json!(2)),
|
||||
("bucket", serde_json::json!("different-bucket")),
|
||||
("base_prefix", serde_json::json!("other/")),
|
||||
("incarnation_id", serde_json::json!(Uuid::new_v4())),
|
||||
("attempt_prefix", serde_json::json!(attempt_prefix())),
|
||||
("object", serde_json::json!(format!("{prefix}other.png"))),
|
||||
("version_id", serde_json::json!(Uuid::new_v4())),
|
||||
("unexpected_field", serde_json::json!(true)),
|
||||
] {
|
||||
let mut changed = decoded.clone();
|
||||
changed
|
||||
.as_object_mut()
|
||||
.expect("strict identity record is an object")
|
||||
.insert(field.to_owned(), value);
|
||||
invalid.push(serde_json::to_vec(&changed).expect("encode invalid persisted identity"));
|
||||
}
|
||||
let (_, missing) = remove_shard(&store, bucket, &object, version).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
for bytes in invalid {
|
||||
// Inject corrupt persisted bytes through real guarded config I/O; the
|
||||
// fixture does not emulate the production retirement state machine.
|
||||
com::with_config_object_write_lock(store.clone(), record.clone(), {
|
||||
let store = store.clone();
|
||||
let record = record.clone();
|
||||
move |guard| async move { com::save_config_no_lock(store, &record, bytes, &guard).await }
|
||||
})
|
||||
.await
|
||||
.expect("hold matching record guard")
|
||||
.expect("persist invalid record on real disks");
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"corrupt or wrong-identity record cannot confirm idempotent retirement"
|
||||
);
|
||||
assert_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert!(!missing.exists(), "invalid record must not authorize real repair publication");
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_waits_for_cancelled_physical_repair_and_retries_cancelled_retirement() {
|
||||
run_large_stack_async_test("artifact-retirement-physical-tail", || async {
|
||||
let root = tempfile::tempdir().expect("physical retirement fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-physical-tail";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let (disk, part) = remove_shard(&store, bucket, &object, version).await;
|
||||
let destination = object_path(&disk, bucket, &object);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let repair_hook = os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
// Sender drop also releases physical work when an assertion unwinds.
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let repair_store = store.clone();
|
||||
let repair_object = object.clone();
|
||||
let mut repair = tokio::spawn(async move {
|
||||
HealOperations::heal_object(repair_store.as_ref(), bucket, &repair_object, &version.to_string(), &heal_options())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
entered = entered_rx => entered.expect("repair reaches real physical rename syscall"),
|
||||
result = &mut repair => panic!("repair returned before syscall pause: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("real repair worker is paused");
|
||||
repair.abort();
|
||||
assert!(repair.await.expect_err("cancel repair caller").is_cancelled());
|
||||
|
||||
let retire_store = store.clone();
|
||||
let retire_prefix = prefix.clone();
|
||||
let retire_object = object.clone();
|
||||
let mut retirement = tokio::spawn(async move {
|
||||
retire_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retire_prefix, &retire_object, version)
|
||||
.await
|
||||
});
|
||||
let blocked = tokio::time::timeout(Duration::from_millis(200), &mut retirement)
|
||||
.await
|
||||
.is_err();
|
||||
drop(release_tx);
|
||||
drop(
|
||||
tokio::time::timeout(DEADLINE, os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination))
|
||||
.await
|
||||
.expect("cancelled physical repair drains"),
|
||||
);
|
||||
drop(repair_hook);
|
||||
if blocked {
|
||||
tokio::time::timeout(DEADLINE, retirement)
|
||||
.await
|
||||
.expect("retirement resumes after real repair tail")
|
||||
.expect("retirement task joins")
|
||||
.expect("retirement persists");
|
||||
}
|
||||
assert!(
|
||||
blocked,
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: retirement must wait for cancelled repair's physical publication tail"
|
||||
);
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
// A cancelled repair may roll back its destination; retirement must stop
|
||||
// any later repair regardless of whether that first publication committed.
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
let part_before = file_bytes(&part).await;
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
assert_eq!(file_bytes(&part).await, part_before);
|
||||
|
||||
let second_prefix = attempt_prefix();
|
||||
let second_object = format!("{second_prefix}cancelled.bin");
|
||||
let second_version = put_payload(&store, bucket, &second_object, 0xB6, false).await;
|
||||
seal(&store, bucket, incarnation, &second_prefix).await;
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &second_prefix, &second_object, second_version);
|
||||
let record_disk = disks(&store).await.remove(0);
|
||||
let record_path = object_path(&record_disk, RUSTFS_META_BUCKET, &record).join(STORAGE_FORMAT_FILE);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _retirement_hook = os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
&record_path,
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let retire_store = store.clone();
|
||||
let retire_prefix = second_prefix.clone();
|
||||
let retire_object = second_object.clone();
|
||||
let mut retirement = tokio::spawn(async move {
|
||||
retire_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retire_prefix, &retire_object, second_version)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
entered = entered_rx => entered.expect("retirement reaches real record rename syscall"),
|
||||
result = &mut retirement => panic!("retirement returned before durable record publication: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retirement's real config publication is paused");
|
||||
retirement.abort();
|
||||
assert!(retirement.await.expect_err("cancel retirement caller").is_cancelled());
|
||||
let retry_store = store.clone();
|
||||
let retry_prefix = second_prefix.clone();
|
||||
let retry_object = second_object.clone();
|
||||
let mut retry = tokio::spawn(async move {
|
||||
retry_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retry_prefix, &retry_object, second_version)
|
||||
.await
|
||||
});
|
||||
let retry_blocked = tokio::time::timeout(Duration::from_millis(200), &mut retry).await.is_err();
|
||||
drop(release_tx);
|
||||
if retry_blocked {
|
||||
tokio::time::timeout(DEADLINE, retry)
|
||||
.await
|
||||
.expect("retry waits for original publication tail")
|
||||
.expect("retry task joins")
|
||||
.expect("cancelled retirement is confirmed idempotently");
|
||||
}
|
||||
assert!(retry_blocked, "retirement retry cannot pass a physically incomplete original publication");
|
||||
let persisted = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("cancelled retirement left durable record");
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &second_prefix, &second_object, second_version)
|
||||
.await
|
||||
.expect("idempotently confirm cancelled retirement");
|
||||
assert_eq!(com::read_config(store.clone(), &record).await.expect("read confirmed record"), persisted);
|
||||
assert_retired_repair_rejected(&store, bucket, &second_object, &second_version.to_string()).await;
|
||||
assert_payload(&store, bucket, &second_object, second_version, 0xB6).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,944 @@
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::metadata_sys,
|
||||
disk::DiskAPI as _,
|
||||
error::Error,
|
||||
object_api::{ObjectInfo, ObjectOptions, PutObjReader, WriteCommitGuard, WriteCompletion},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
multipart::{CompletePart, MultipartOperations as _},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{collections::HashMap, time::Duration};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn object_key() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/image.png", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn options() -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn body() -> PutObjReader {
|
||||
PutObjReader::from_vec(vec![0xA5; 273])
|
||||
}
|
||||
|
||||
fn unsupported<T>(result: Result<T, Error>) {
|
||||
let Err(error) = result else {
|
||||
panic!("ARTIFACT_WRITE_UNSUPPORTED: operation unexpectedly succeeded")
|
||||
};
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket does not support this write operation"),
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn bucket(store: &ECStore, name: &str, artifact: bool) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
name,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: artifact.then(|| "outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create real versioned fixture bucket");
|
||||
store.bucket_incarnation_id(name).await.expect("persisted incarnation")
|
||||
}
|
||||
|
||||
async fn snapshot(store: &ECStore, bucket: &str, object: &str) -> Vec<Vec<u8>> {
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
assert_eq!(disks.len(), 4);
|
||||
let mut versions = Vec::new();
|
||||
for disk in disks {
|
||||
versions.push(
|
||||
disk.expect("online fixture disk")
|
||||
.read_xl(bucket, object, false)
|
||||
.await
|
||||
.expect("physical version metadata")
|
||||
.buf,
|
||||
);
|
||||
}
|
||||
versions
|
||||
}
|
||||
|
||||
async fn copy_info(store: &ECStore, bucket: &str, object: &str) -> ObjectInfo {
|
||||
let mut info = store
|
||||
.get_object_info(bucket, object, &options())
|
||||
.await
|
||||
.expect("real source object");
|
||||
info.put_object_reader = Some(body());
|
||||
info
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_object_mutations_are_rejected() {
|
||||
run_large_stack_async_test("artifact-object-entrypoints", object_mutations_case);
|
||||
}
|
||||
|
||||
async fn object_mutations_case() {
|
||||
let root = tempfile::tempdir().expect("object entrypoint fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-object-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let artifact = "artifact-entrypoint-object";
|
||||
let ordinary = "artifact-entrypoint-control";
|
||||
bucket(&store, artifact, true).await;
|
||||
bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary owned artifact PUT is supported");
|
||||
store
|
||||
.put_object(ordinary, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary control object");
|
||||
let before = snapshot(&store, artifact, &object).await;
|
||||
|
||||
let mut source = copy_info(&store, ordinary, &object).await;
|
||||
unsupported(
|
||||
store
|
||||
.copy_object(ordinary, &object, artifact, &object, &mut source, &options(), &options())
|
||||
.await,
|
||||
);
|
||||
let mut source = copy_info(&store, artifact, &object).await;
|
||||
unsupported(
|
||||
store
|
||||
.copy_object(artifact, &object, artifact, &object, &mut source, &options(), &options())
|
||||
.await,
|
||||
);
|
||||
for opts in [
|
||||
ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
data_movement: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
preserve_ciphertext: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
metadata_chg: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
replication_status_writeback: Some(Box::default()),
|
||||
..options()
|
||||
},
|
||||
] {
|
||||
unsupported(store.put_object(artifact, &object, &mut body(), &opts).await);
|
||||
}
|
||||
unsupported(
|
||||
store
|
||||
.put_object_for_data_movement(
|
||||
artifact,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
..options()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
{
|
||||
let lock = store
|
||||
.new_ns_lock(artifact, &object)
|
||||
.await
|
||||
.expect("real borrowed target namespace");
|
||||
let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30))
|
||||
.await
|
||||
.expect("real borrowed write owner");
|
||||
let mut opts = options();
|
||||
opts.add_write_commit_guard(&guard);
|
||||
unsupported(store.put_object(artifact, &object, &mut body(), &opts).await);
|
||||
}
|
||||
unsupported(store.transition_object(artifact, &object, &options()).await);
|
||||
unsupported(store.clone().restore_transitioned_object(artifact, &object, &options()).await);
|
||||
unsupported(
|
||||
store
|
||||
.put_object_metadata(
|
||||
artifact,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("x-amz-meta-test".to_owned(), "changed".to_owned())])),
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
);
|
||||
unsupported(store.put_object_tags(artifact, &object, "kind=artifact", &options()).await);
|
||||
unsupported(store.delete_object_tags(artifact, &object, &options()).await);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: every disk retains the exact original versions"
|
||||
);
|
||||
|
||||
// An artifact source remains readable; only the destination write is classified.
|
||||
let mut source = copy_info(&store, artifact, &object).await;
|
||||
store
|
||||
.copy_object(artifact, &object, ordinary, "copied.png", &mut source, &options(), &options())
|
||||
.await
|
||||
.expect("copy into ordinary bucket stays supported");
|
||||
store
|
||||
.put_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary replication PUT stays supported");
|
||||
{
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(ordinary)
|
||||
.await
|
||||
.expect("ordinary incarnation");
|
||||
let lifecycle = std::sync::Arc::new(
|
||||
store
|
||||
.acquire_bucket_incarnation_fence(ordinary, incarnation)
|
||||
.await
|
||||
.expect("ordinary lifecycle owner"),
|
||||
);
|
||||
let lock = store.new_ns_lock(ordinary, &object).await.expect("ordinary target namespace");
|
||||
let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30))
|
||||
.await
|
||||
.expect("ordinary borrowed owner");
|
||||
let mut opts = options();
|
||||
lifecycle.attach_owner(&mut opts);
|
||||
opts.add_write_commit_guard(&guard);
|
||||
store
|
||||
.put_object(ordinary, &object, &mut body(), &opts)
|
||||
.await
|
||||
.expect("ordinary borrowed PUT stays supported");
|
||||
}
|
||||
store
|
||||
.put_object_metadata(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("x-amz-meta-test".to_owned(), "changed".to_owned())])),
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary metadata update stays supported");
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "kind=ordinary", &options())
|
||||
.await
|
||||
.expect("ordinary tag update stays supported");
|
||||
store
|
||||
.delete_object_tags(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary tag removal stays supported");
|
||||
assert_eq!(
|
||||
store
|
||||
.get_object_info(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary object remains readable")
|
||||
.size,
|
||||
273
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: reading an artifact source cannot mutate its versions"
|
||||
);
|
||||
let tier = "ARTIFACT-ENTRYPOINT-CONTROL";
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&ctx.tier_config_mgr(), tier).await;
|
||||
let current = store
|
||||
.get_object_info(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary source before transition");
|
||||
let exact = ObjectOptions {
|
||||
version_id: current.version_id.map(|version| version.to_string()),
|
||||
..options()
|
||||
};
|
||||
store
|
||||
.transition_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
transition: crate::bucket::lifecycle::lifecycle::TransitionOptions {
|
||||
status: rustfs_filemeta::TRANSITION_PENDING.to_owned(),
|
||||
tier: tier.to_owned(),
|
||||
etag: current.etag.expect("ordinary source etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: current.mod_time,
|
||||
..exact.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary transition stays supported");
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
let mut restore_transition = crate::bucket::lifecycle::lifecycle::TransitionOptions::default();
|
||||
restore_transition.restore_request.days = Some(1);
|
||||
store
|
||||
.clone()
|
||||
.restore_transitioned_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
transition: restore_transition,
|
||||
..exact
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary restore stays supported");
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_multipart_rejects_writes_but_keeps_legacy_cleanup() {
|
||||
run_large_stack_async_test("artifact-multipart-entrypoints", multipart_case);
|
||||
}
|
||||
|
||||
async fn multipart_case() {
|
||||
let root = tempfile::tempdir().expect("multipart entrypoint fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-multipart-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let artifact = "artifact-entrypoint-multipart";
|
||||
let ordinary = "artifact-multipart-control";
|
||||
let incarnation = bucket(&store, artifact, true).await;
|
||||
bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("seed the current artifact version");
|
||||
let before = snapshot(&store, artifact, &object).await;
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..options()
|
||||
};
|
||||
unsupported(store.new_multipart_upload(artifact, &object, &opts).await);
|
||||
// Seed actual legacy staging while holding the lifecycle owner required by the pool API.
|
||||
let lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock(artifact)
|
||||
.await
|
||||
.expect("legacy upload lifecycle owner");
|
||||
let mut staging_opts = opts.clone();
|
||||
staging_opts.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(artifact, &object, &staging_opts)
|
||||
.await
|
||||
.expect("real legacy upload staging");
|
||||
let part = store.pools[0]
|
||||
.put_object_part(artifact, &object, &upload.upload_id, 1, &mut body(), &staging_opts)
|
||||
.await
|
||||
.expect("real legacy staged part");
|
||||
drop(staging_opts);
|
||||
drop(lifecycle);
|
||||
let parts = vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag.clone(),
|
||||
..Default::default()
|
||||
}];
|
||||
unsupported(
|
||||
store
|
||||
.put_object_part(artifact, &object, &upload.upload_id, 2, &mut body(), &opts)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(artifact, &object, &upload.upload_id, parts.clone(), &opts)
|
||||
.await,
|
||||
);
|
||||
let movement = ObjectOptions {
|
||||
data_movement: true,
|
||||
part_number: Some(2),
|
||||
..opts.clone()
|
||||
};
|
||||
unsupported(
|
||||
store
|
||||
.handle_new_multipart_upload_with_pool_idx(artifact, &object, &movement, None)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.put_object_part_for_data_movement(0, artifact, &object, &upload.upload_id, &mut body(), &movement)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload_for_data_movement((0, None), artifact, &object, &upload.upload_id, parts, &movement)
|
||||
.await,
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: rejected multipart completion publishes no version"
|
||||
);
|
||||
let listing = store
|
||||
.list_multipart_uploads(artifact, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("list legacy uploads remains supported");
|
||||
assert_eq!(listing.uploads.len(), 1);
|
||||
assert_eq!(listing.uploads[0].upload_id, upload.upload_id);
|
||||
let staged = store
|
||||
.list_object_parts(artifact, &object, &upload.upload_id, None, 100, &opts)
|
||||
.await
|
||||
.expect("list legacy parts remains supported");
|
||||
assert_eq!(staged.parts.len(), 1);
|
||||
assert_eq!(staged.parts[0].etag, part.etag);
|
||||
store
|
||||
.abort_multipart_upload(artifact, &object, &upload.upload_id, &opts)
|
||||
.await
|
||||
.expect("abort legacy upload remains supported");
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(artifact, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("list after abort")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: legacy abort leaves published versions intact"
|
||||
);
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary multipart initialization");
|
||||
let part = store
|
||||
.put_object_part(ordinary, &object, &upload.upload_id, 1, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary part write");
|
||||
let completed = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
ordinary,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&options(),
|
||||
)
|
||||
.await
|
||||
.expect("ordinary multipart completion");
|
||||
assert_eq!(completed.size, 273);
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_classification_is_bound_to_bucket_incarnation() {
|
||||
run_large_stack_async_test("artifact-entrypoint-recreation", recreation_case);
|
||||
}
|
||||
|
||||
async fn recreation_case() {
|
||||
let root = tempfile::tempdir().expect("entrypoint recreation fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-entrypoint-recreation", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let name = "artifact-entrypoint-recreated";
|
||||
let old_incarnation = bucket(&store, name, false).await;
|
||||
let object = object_key();
|
||||
let barrier =
|
||||
crate::set_disk::PutObjectCommitBarrier::install(name, &object, crate::set_disk::PutObjectCommitPause::BeforeNamespace);
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
name,
|
||||
&put_object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::time::timeout(Duration::from_secs(30), store.delete_bucket(name, &DeleteBucketOptions::default()))
|
||||
.await
|
||||
.expect("staged old-generation upload does not prevent bucket deletion")
|
||||
.expect("delete old ordinary bucket");
|
||||
let new_incarnation = bucket(&store, name, true).await;
|
||||
assert_ne!(new_incarnation, old_incarnation);
|
||||
barrier.release();
|
||||
let error = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("stale staged PUT terminates")
|
||||
.expect("PUT task joins")
|
||||
.expect_err("classification of an ordinary bucket cannot authorize the new artifact bucket");
|
||||
assert!(
|
||||
matches!(error, Error::BucketNotFound(ref bucket) if bucket == name),
|
||||
"ARTIFACT_WRITE_INCARNATION: {error}"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let error = disk
|
||||
.expect("online fixture disk")
|
||||
.read_xl(name, &object, false)
|
||||
.await
|
||||
.err()
|
||||
.expect("no stale object metadata may be published");
|
||||
assert!(
|
||||
matches!(error, crate::disk::error::DiskError::FileNotFound),
|
||||
"ARTIFACT_WRITE_INCARNATION: {error}"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_lifecycle_fences_require_real_matching_owners() {
|
||||
run_large_stack_async_test("artifact-entrypoint-lifecycle-owner", lifecycle_owner_case);
|
||||
}
|
||||
|
||||
async fn lifecycle_owner_case() {
|
||||
let root = tempfile::tempdir().expect("lifecycle owner fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-lifecycle-owner", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let ordinary = "artifact-lifecycle-ordinary";
|
||||
let artifact = "artifact-lifecycle-dedicated";
|
||||
bucket(&store, ordinary, false).await;
|
||||
bucket(&store, artifact, true).await;
|
||||
let object = object_key();
|
||||
for name in [ordinary, artifact] {
|
||||
store
|
||||
.put_object(name, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("initial real object");
|
||||
}
|
||||
let before = snapshot(&store, ordinary, &object).await;
|
||||
let artifact_before = snapshot(&store, artifact, &object).await;
|
||||
let unrelated = store
|
||||
.new_ns_lock(ordinary, "unrelated-object")
|
||||
.await
|
||||
.expect("unrelated namespace");
|
||||
let unrelated_owner = unrelated
|
||||
.get_read_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("live unrelated owner");
|
||||
let mut foreign_fence = options();
|
||||
foreign_fence.add_bucket_lifecycle_lock_guard(&unrelated_owner);
|
||||
let lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock(ordinary)
|
||||
.await
|
||||
.expect("real lifecycle owner");
|
||||
let mut released_fence = options();
|
||||
released_fence.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
let borrowed_snapshot = store
|
||||
.object_lock_config_snapshot_under_lifecycle_fence(
|
||||
ordinary,
|
||||
released_fence
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.expect("borrowed lifecycle fence"),
|
||||
)
|
||||
.await
|
||||
.expect("real snapshot retaining only its metadata lock");
|
||||
drop(lifecycle);
|
||||
let mut snapshot_without_owner = options();
|
||||
borrowed_snapshot.add_lock_fences(&mut snapshot_without_owner);
|
||||
snapshot_without_owner.object_lock_config_snapshot = Some(borrowed_snapshot);
|
||||
|
||||
for opts in [foreign_fence, released_fence, snapshot_without_owner] {
|
||||
let mut source = copy_info(&store, ordinary, &object).await;
|
||||
for result in [
|
||||
store
|
||||
.put_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store
|
||||
.copy_object(ordinary, &object, ordinary, &object, &mut source, &options(), &opts)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "owner=fake", &opts)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store.put_object_metadata(ordinary, &object, &opts).await.map(|_| ()),
|
||||
store.clone().restore_transitioned_object(ordinary, &object, &opts).await,
|
||||
store.new_multipart_upload(ordinary, &object, &opts).await.map(|_| ()),
|
||||
] {
|
||||
let error = result.expect_err("a fence without its matching lifecycle owner cannot authorize a write");
|
||||
assert!(
|
||||
error.to_string().contains("write requires an owned bucket lifecycle fence"),
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: {error}"
|
||||
);
|
||||
}
|
||||
let error = store
|
||||
.put_object(artifact, &object, &mut body(), &opts)
|
||||
.await
|
||||
.expect_err("ordinary artifact PUT also requires the lifecycle owner");
|
||||
assert!(
|
||||
error.to_string().contains("write requires an owned bucket lifecycle fence"),
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, ordinary, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: ordinary metadata unchanged"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
artifact_before,
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: artifact versions unchanged"
|
||||
);
|
||||
}
|
||||
drop(unrelated_owner);
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(ordinary, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("inspect rejected multipart creation")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let owned_snapshot = store
|
||||
.object_lock_config_snapshot(ordinary)
|
||||
.await
|
||||
.expect("real lifecycle-owning snapshot");
|
||||
let mut owned = options();
|
||||
owned_snapshot.add_lock_fences(&mut owned);
|
||||
owned.object_lock_config_snapshot = Some(owned_snapshot.clone());
|
||||
let lifecycle_namespace = store
|
||||
.new_ns_lock(ordinary, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||
.await
|
||||
.expect("actual bucket deletion namespace");
|
||||
assert!(
|
||||
lifecycle_namespace.get_write_lock(Duration::from_millis(100)).await.is_err(),
|
||||
"matching snapshot actually blocks the bucket deletion lock"
|
||||
);
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "owner=real", &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary tags");
|
||||
let upload = store
|
||||
.new_multipart_upload(ordinary, &object, &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary multipart creation");
|
||||
store
|
||||
.abort_multipart_upload(ordinary, &object, &upload.upload_id, &owned)
|
||||
.await
|
||||
.expect("clean up supported multipart upload");
|
||||
drop(owned);
|
||||
drop(owned_snapshot);
|
||||
let artifact_snapshot = store
|
||||
.object_lock_config_snapshot(artifact)
|
||||
.await
|
||||
.expect("artifact lifecycle-owning snapshot");
|
||||
let mut owned = options();
|
||||
artifact_snapshot.add_lock_fences(&mut owned);
|
||||
owned.object_lock_config_snapshot = Some(artifact_snapshot);
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary artifact PUT");
|
||||
assert_ne!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
artifact_before,
|
||||
"supported artifact PUT publishes a new version"
|
||||
);
|
||||
drop(owned);
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
// Include directories as well as bytes: a rejected delete must not even stage
|
||||
// a rollback directory before another layer can refuse the request.
|
||||
async fn delete_physical_snapshot(
|
||||
store: &ECStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Vec<std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>> {
|
||||
let mut snapshots = Vec::new();
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let disk = disk.expect("online delete fixture disk");
|
||||
let root = disk.path().join(bucket).join(object);
|
||||
let mut entries = std::collections::BTreeMap::new();
|
||||
let mut pending = vec![root.clone()];
|
||||
while let Some(directory) = pending.pop() {
|
||||
let mut children = tokio::fs::read_dir(&directory)
|
||||
.await
|
||||
.expect("read physical artifact directory");
|
||||
while let Some(child) = children.next_entry().await.expect("read physical artifact entry") {
|
||||
let path = child.path();
|
||||
let relative = path.strip_prefix(&root).expect("fixture descendant").to_owned();
|
||||
if child.file_type().await.expect("physical entry type").is_dir() {
|
||||
entries.insert(relative, None);
|
||||
pending.push(path);
|
||||
} else {
|
||||
entries.insert(relative, Some(tokio::fs::read(path).await.expect("physical artifact bytes")));
|
||||
}
|
||||
}
|
||||
}
|
||||
snapshots.push(entries);
|
||||
}
|
||||
snapshots
|
||||
}
|
||||
|
||||
async fn seed_delete_versions(store: &ECStore, bucket: &str, object: &str, incarnation: Uuid) -> (Uuid, Uuid) {
|
||||
let first = store
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0xA5; 2 * 1024 * 1024]), &options())
|
||||
.await
|
||||
.expect("publish actual external artifact shards")
|
||||
.version_id
|
||||
.expect("versioned artifact identity");
|
||||
store
|
||||
.put_object(bucket, object, &mut body(), &options())
|
||||
.await
|
||||
.expect("publish another readable artifact version");
|
||||
let marker = Uuid::new_v4();
|
||||
let mut fi = rustfs_filemeta::FileInfo {
|
||||
name: object.to_owned(),
|
||||
version_id: Some(marker),
|
||||
deleted: true,
|
||||
mod_time: Some(time::OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
fi.set_delete_marker_incarnation(incarnation);
|
||||
// Seed a pre-existing marker through the physical fixture API. Ordinary
|
||||
// managed deletion must never be needed to construct its own refusal case.
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
disk.expect("online marker fixture disk")
|
||||
.write_metadata("", bucket, object, fi.clone())
|
||||
.await
|
||||
.expect("persist pre-existing delete marker");
|
||||
}
|
||||
(first, marker)
|
||||
}
|
||||
|
||||
fn delete_variants(version: Uuid, marker: Uuid) -> Vec<ObjectOptions> {
|
||||
vec![
|
||||
options(),
|
||||
ObjectOptions {
|
||||
version_id: Some(version.to_string()),
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: Some(marker.to_string()),
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
delete_marker: true,
|
||||
replication_request: true,
|
||||
delete_replication: Some(crate::bucket::replication::ReplicationState::default()),
|
||||
..options()
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_delete_entrypoints_store_rejects_single_and_batch_before_mutation() {
|
||||
run_large_stack_async_test("artifact-delete-store", || delete_entrypoints_case(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_delete_entrypoints_set_disks_rejects_direct_mutations() {
|
||||
run_large_stack_async_test("artifact-delete-set-disks", || delete_entrypoints_case(true));
|
||||
}
|
||||
|
||||
async fn delete_entrypoints_case(direct: bool) {
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
let root = tempfile::tempdir().expect("delete entrypoint fixture root");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-delete-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let managed = "artifact-delete-managed";
|
||||
let ordinary = "artifact-delete-control";
|
||||
let incarnation = bucket(&store, managed, true).await;
|
||||
let ordinary_incarnation = bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
let (version, marker) = seed_delete_versions(&store, managed, &object, incarnation).await;
|
||||
let (control_version, control_marker) = seed_delete_versions(&store, ordinary, &object, ordinary_incarnation).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let before = delete_physical_snapshot(&store, managed, &object).await;
|
||||
assert!(
|
||||
before.iter().all(|disk| disk.keys().any(|path| path.ends_with("part.1"))),
|
||||
"fixture must include external shards"
|
||||
);
|
||||
|
||||
for sealed in [false, true] {
|
||||
if sealed {
|
||||
let attempt = object.rsplit_once('/').expect("attempt/object fixture").0.to_owned() + "/";
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(managed, incarnation, &attempt)
|
||||
.await
|
||||
.expect("seal ordinary PUT domain");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_repairs(managed, incarnation, &attempt)
|
||||
.await
|
||||
.expect("seal repair domain");
|
||||
}
|
||||
for opts in delete_variants(version, marker) {
|
||||
let result = if direct {
|
||||
set.delete_object(managed, &object, opts).await
|
||||
} else {
|
||||
// Hold the object lock to distinguish the ECStore guard from a
|
||||
// later SetDisks refusal, without removing that second defense.
|
||||
let lock = store.new_ns_lock(managed, &object).await.expect("delete namespace");
|
||||
let _held = lock
|
||||
.get_write_lock(Duration::from_secs(5))
|
||||
.await
|
||||
.expect("hold delete namespace");
|
||||
tokio::time::timeout(Duration::from_secs(1), store.delete_object(managed, &object, opts))
|
||||
.await
|
||||
.expect("managed rejection precedes object locking")
|
||||
};
|
||||
unsupported(result);
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
}
|
||||
let targets = vec![
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(marker),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker_replication_status: Some("REPLICA".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let (deleted, errors) = if direct {
|
||||
set.delete_objects(managed, targets.clone(), options()).await
|
||||
} else {
|
||||
let lock = store.new_ns_lock(managed, &object).await.expect("batch namespace");
|
||||
let _held = lock
|
||||
.get_write_lock(Duration::from_secs(5))
|
||||
.await
|
||||
.expect("hold batch namespace");
|
||||
tokio::time::timeout(Duration::from_secs(1), store.delete_objects(managed, targets.clone(), options()))
|
||||
.await
|
||||
.expect("managed batch rejection precedes object locking")
|
||||
};
|
||||
assert_eq!(deleted.len(), targets.len());
|
||||
assert_eq!(errors.len(), targets.len());
|
||||
for error in errors {
|
||||
unsupported::<()>(Err(error.expect("each managed delete retains its own refusal")));
|
||||
}
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
let request = rustfs_filemeta::FileInfo {
|
||||
version_id: Some(version),
|
||||
name: object.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
unsupported(if direct {
|
||||
set.delete_object_version(managed, &object, &request, false).await
|
||||
} else {
|
||||
store.delete_object_version(managed, &object, &request, false).await
|
||||
});
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
}
|
||||
|
||||
let control_before = delete_physical_snapshot(&store, ordinary, &object).await;
|
||||
let mut control_opts = options();
|
||||
if direct {
|
||||
// An unbound policy snapshot remains supported, but is not lifecycle
|
||||
// authority. The direct entry point must acquire its own real owner.
|
||||
control_opts.object_lock_config_snapshot = Some(std::sync::Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
|
||||
metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
)));
|
||||
}
|
||||
let created = if direct {
|
||||
set.delete_object(ordinary, &object, control_opts.clone()).await
|
||||
} else {
|
||||
store.delete_object(ordinary, &object, control_opts.clone()).await
|
||||
}
|
||||
.expect("ordinary bucket actually creates a delete marker");
|
||||
assert!(created.delete_marker);
|
||||
assert_ne!(delete_physical_snapshot(&store, ordinary, &object).await, control_before);
|
||||
let control_targets = vec![
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(control_version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(control_marker),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let (_, errors) = if direct {
|
||||
set.delete_objects(ordinary, control_targets, control_opts).await
|
||||
} else {
|
||||
store.delete_objects(ordinary, control_targets, control_opts).await
|
||||
};
|
||||
assert!(errors.iter().all(Option::is_none), "ordinary batch deletion must commit: {errors:?}");
|
||||
for disk in set.disk_inventory().await {
|
||||
let disk = disk.expect("online control disk");
|
||||
for removed in [control_version, control_marker] {
|
||||
assert!(
|
||||
disk.read_version(ordinary, ordinary, &object, &removed.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"ordinary deletion removed the exact physical version"
|
||||
);
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -86,6 +86,160 @@ pub async fn connect_load_init_formats(
|
||||
pub(crate) struct LoadedFormat {
|
||||
pub(crate) format: FormatV3,
|
||||
pub(crate) pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority,
|
||||
pub(crate) pending_disks: Vec<PendingFormatDisk>,
|
||||
}
|
||||
|
||||
pub(crate) struct PendingFormatDisk {
|
||||
disk: DiskStore,
|
||||
}
|
||||
|
||||
impl PendingFormatDisk {
|
||||
/// Reconcile only after bootstrap metadata has succeeded, before Ready.
|
||||
pub(crate) async fn finalize(
|
||||
&self,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
sets: &[Arc<crate::set_disk::SetDisks>],
|
||||
) -> Result<()> {
|
||||
let endpoint = self.disk.endpoint();
|
||||
let pool_index = usize::try_from(endpoint.pool_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set_index = usize::try_from(endpoint.set_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let disk_index = usize::try_from(endpoint.disk_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set = sets
|
||||
.iter()
|
||||
.find(|set| set.pool_index == pool_index && set.set_index == set_index)
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if set.set_endpoints.get(disk_index) != Some(&endpoint) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
// Capture current handles, including a reconnect that completed since
|
||||
// format selection. No registry guard is held during format I/O.
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let receivers = registry.read().await;
|
||||
let receiver = if endpoint.is_local {
|
||||
receivers.get(&endpoint.to_string()).and_then(Clone::clone)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let selected = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.get(disk_index)
|
||||
.cloned()
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if endpoint.is_local && receiver.is_none() {
|
||||
// An earlier successful pass may already have quarantined this
|
||||
// slot before a different slot raced with reconnect and retried.
|
||||
let topology = instance_ctx.local_disk_set_drives();
|
||||
let local_slots = topology.read().await;
|
||||
let local_slot = local_slots
|
||||
.get(pool_index)
|
||||
.and_then(|sets| sets.get(set_index))
|
||||
.and_then(|disks| disks.get(disk_index))
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
return if matches!(receivers.get(&endpoint.to_string()), Some(None)) && local_slot.is_none() && selected.is_none() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::other("format reconciliation receiver changed"))
|
||||
};
|
||||
}
|
||||
drop(receivers);
|
||||
let disk = selected.as_ref().unwrap_or(&self.disk);
|
||||
let observation = load_format_erasure(receiver.as_ref().unwrap_or(disk), false).await;
|
||||
self.publish_observation(instance_ctx, set, selected.as_ref(), receiver.as_ref(), observation)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn publish_observation(
|
||||
&self,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
set: &crate::set_disk::SetDisks,
|
||||
selected: Option<&DiskStore>,
|
||||
receiver: Option<&DiskStore>,
|
||||
observation: disk::error::Result<FormatV3>,
|
||||
) -> Result<()> {
|
||||
let endpoint = self.disk.endpoint();
|
||||
let pool_index = usize::try_from(endpoint.pool_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set_index = usize::try_from(endpoint.set_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let disk_index = usize::try_from(endpoint.disk_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
if set.pool_index != pool_index || set.set_index != set_index || set.set_endpoints.get(disk_index) != Some(&endpoint) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
let valid = observation.as_ref().is_ok_and(|format| {
|
||||
set.format.check_other(format).is_ok()
|
||||
&& set.format.find_disk_index_by_disk_id(format.erasure.this).ok() == Some((set_index, disk_index))
|
||||
});
|
||||
let disk_id = if valid {
|
||||
Some(observation.map_err(Error::from)?.erasure.this)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let path = endpoint.to_string();
|
||||
// Local sets must share the registered receiver's health and I/O
|
||||
// admission state, just as Sets::new_with_instance_ctx does.
|
||||
let disk = receiver.unwrap_or_else(|| selected.unwrap_or(&self.disk));
|
||||
if disk.endpoint() != endpoint
|
||||
|| endpoint.is_local != receiver.is_some()
|
||||
|| receiver.is_some_and(|receiver| receiver.endpoint() != endpoint)
|
||||
{
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
|
||||
// All format I/O is complete. Publish under registry -> ID map -> local
|
||||
// topology -> set slot, rechecking handle identity before any mutation.
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let mut receivers = registry.write().await;
|
||||
let id_map = instance_ctx.local_disk_id_map();
|
||||
let mut ids = id_map.write().await;
|
||||
let topology = instance_ctx.local_disk_set_drives();
|
||||
let mut local_slots = topology.write().await;
|
||||
let mut slots = set.disks.write().await;
|
||||
let slot = slots.get_mut(disk_index).ok_or(Error::CorruptedFormat)?;
|
||||
let unchanged = match (slot.as_ref(), selected) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(observed)) => Arc::ptr_eq(current, observed),
|
||||
_ => false,
|
||||
};
|
||||
if !unchanged {
|
||||
return Err(Error::other("format reconciliation slot changed"));
|
||||
}
|
||||
if let Some(receiver) = receiver {
|
||||
if !receivers
|
||||
.get(&path)
|
||||
.and_then(Option::as_ref)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, receiver))
|
||||
{
|
||||
return Err(Error::other("format reconciliation receiver changed"));
|
||||
}
|
||||
let local_slot = local_slots
|
||||
.get_mut(pool_index)
|
||||
.and_then(|sets| sets.get_mut(set_index))
|
||||
.and_then(|disks| disks.get_mut(disk_index))
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if !local_slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, receiver)) {
|
||||
return Err(Error::other("format reconciliation local slot changed"));
|
||||
}
|
||||
// Local ID setters only update memory, without filesystem I/O.
|
||||
receiver.set_disk_id_state(disk_id).await?;
|
||||
ids.retain(|_, registered_path| registered_path != &path);
|
||||
if let Some(id) = disk_id {
|
||||
ids.insert(id, path.clone());
|
||||
} else {
|
||||
receivers.insert(path, None);
|
||||
*local_slot = None;
|
||||
}
|
||||
}
|
||||
if receiver.is_none() {
|
||||
disk.set_disk_id_state(disk_id).await?;
|
||||
}
|
||||
if valid {
|
||||
disk.enable_health_check();
|
||||
*slot = Some(disk.clone());
|
||||
} else {
|
||||
*slot = None;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
@@ -137,13 +291,15 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &members, set_drive_count).await?;
|
||||
let pending_disks =
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &members, set_drive_count, Some(&errs)).await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
pending_disks,
|
||||
});
|
||||
}
|
||||
let resumable_partial_migration = formats_present > 0
|
||||
@@ -158,10 +314,11 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
|
||||
info!("Migrated format from MinIO config");
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count, None).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: *format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::LegacyAdoption,
|
||||
pending_disks: Vec::new(),
|
||||
});
|
||||
}
|
||||
Ok(LegacyFormatOutcome::Incompatible) => {
|
||||
@@ -182,6 +339,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
|
||||
pending_disks: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -206,11 +364,13 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
None => select_format_erasure_in_quorum(&formats, 0)?,
|
||||
};
|
||||
check_format_erasure_value_for_topology(&fm, formats.len(), set_drive_count)?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &fm, &quorum_members, set_drive_count).await?;
|
||||
let pending_disks =
|
||||
retain_format_quorum_members(instance_ctx, disks, &fm, &quorum_members, set_drive_count, Some(&errs)).await?;
|
||||
|
||||
Ok(LoadedFormat {
|
||||
format: fm,
|
||||
pool_meta_bootstrap_authority: verified_legacy_adoption_source(disks, &formats, set_count, set_drive_count).await?,
|
||||
pending_disks,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -239,7 +399,8 @@ async fn retain_format_quorum_members(
|
||||
format: &FormatV3,
|
||||
quorum_members: &[bool],
|
||||
set_drive_count: usize,
|
||||
) -> Result<()> {
|
||||
observed_errors: Option<&[Option<DiskError>]>,
|
||||
) -> Result<Vec<PendingFormatDisk>> {
|
||||
if set_drive_count == 0 || quorum_members.len() != disks.len() {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
@@ -279,10 +440,31 @@ async fn retain_format_quorum_members(
|
||||
let mut local_pool_endpoints = Vec::new();
|
||||
let mut selected_local_disk_ids = Vec::new();
|
||||
let mut quarantined_endpoints = Vec::new();
|
||||
let mut pending_disks = Vec::new();
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let receivers = registry.read().await;
|
||||
for (index, ((disk, endpoint), belongs_to_quorum)) in disks.iter().zip(&endpoints).zip(quorum_members).enumerate() {
|
||||
let pending = !belongs_to_quorum
|
||||
&& matches!(
|
||||
observed_errors.and_then(|errors| errors.get(index)),
|
||||
Some(Some(DiskError::UnformattedDisk))
|
||||
);
|
||||
let receiver = endpoint
|
||||
.as_ref()
|
||||
.filter(|endpoint| endpoint.is_local)
|
||||
.and_then(|endpoint| receivers.get(&endpoint.to_string()))
|
||||
.and_then(Clone::clone);
|
||||
// A first writer may still be publishing this slot's format. Keep its
|
||||
// receiver reachable, but do not count it as a format quorum member.
|
||||
let defer = pending && disk.as_ref().is_some_and(|disk| !disk.is_local() || receiver.is_some());
|
||||
if defer {
|
||||
pending_disks.push(PendingFormatDisk {
|
||||
disk: disk.as_ref().ok_or(Error::CorruptedFormat)?.clone(),
|
||||
});
|
||||
}
|
||||
if let Some(endpoint) = endpoint.as_ref().filter(|endpoint| endpoint.is_local) {
|
||||
local_pool_endpoints.push(endpoint.to_string());
|
||||
if !belongs_to_quorum {
|
||||
if !belongs_to_quorum && !defer {
|
||||
quarantined_endpoints.push(endpoint.clone());
|
||||
}
|
||||
}
|
||||
@@ -302,6 +484,7 @@ async fn retain_format_quorum_members(
|
||||
selected_local_disk_ids.push((disk_id, disk.endpoint().to_string()));
|
||||
}
|
||||
}
|
||||
drop(receivers);
|
||||
quarantine_local_disks(instance_ctx, &quarantined_endpoints).await?;
|
||||
|
||||
for (disk, disk_id) in disks.iter().zip(member_disk_ids) {
|
||||
@@ -315,7 +498,7 @@ async fn retain_format_quorum_members(
|
||||
*disk = None;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
Ok(pending_disks)
|
||||
}
|
||||
|
||||
pub fn should_init_erasure_disks(errs: &[Option<DiskError>]) -> bool {
|
||||
@@ -369,7 +552,7 @@ async fn init_format_erasure(
|
||||
write_format_file_all(disks, &fms).await?;
|
||||
let format = get_format_erasure_in_quorum(&fms, 0)?;
|
||||
let quorum_members = vec![true; disks.len()];
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count, None).await?;
|
||||
Ok(format)
|
||||
}
|
||||
|
||||
@@ -976,6 +1159,314 @@ mod tests {
|
||||
(temp_dir, disks)
|
||||
}
|
||||
|
||||
struct PendingFormatFixture {
|
||||
ctx: Arc<InstanceContext>,
|
||||
set: Arc<crate::set_disk::SetDisks>,
|
||||
pending: PendingFormatDisk,
|
||||
receiver: DiskStore,
|
||||
format: FormatV3,
|
||||
_temp: tempfile::TempDir,
|
||||
}
|
||||
|
||||
async fn pending_format_fixture() -> PendingFormatFixture {
|
||||
let (temp, mut disks) = local_disks(4).await;
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let format = FormatV3::new(1, 4);
|
||||
let endpoints = disks.iter().flatten().map(|disk| disk.endpoint()).collect::<Vec<_>>();
|
||||
// Bootstrap receivers are distinct from the subsequently opened set handles.
|
||||
let (receivers, errors) = init_disks(
|
||||
&Endpoints::from(endpoints.clone()),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none));
|
||||
record_local_disks(&ctx, receivers.iter().flatten().cloned().collect()).await;
|
||||
*ctx.local_disk_set_drives().write().await = vec![vec![receivers.clone()]];
|
||||
for (index, disk) in disks.iter().enumerate().take(3) {
|
||||
let mut member = format.clone();
|
||||
member.erasure.this = format.erasure.sets[0][index];
|
||||
save_format_file(disk, &Some(member))
|
||||
.await
|
||||
.expect("first three formats should publish");
|
||||
}
|
||||
let receiver = receivers[3].as_ref().expect("pending receiver should exist").clone();
|
||||
let mut last = format.clone();
|
||||
last.erasure.this = format.erasure.sets[0][3];
|
||||
let bootstrap = crate::store::peer::BootstrapLocalTarget::new(ctx.clone());
|
||||
bootstrap
|
||||
.begin_legacy_local_file_write(&receiver, RUSTFS_META_BUCKET, None)
|
||||
.await
|
||||
.expect("pending format writer should be admitted")
|
||||
.write_all("pending-format", last.to_json().expect("format should encode").into_bytes().into())
|
||||
.await
|
||||
.expect("last format temporary file should be durable");
|
||||
let mut loaded = connect_load_init_formats_with_instance_ctx(&ctx, false, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("three existing formats should supply read quorum");
|
||||
assert!(matches!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None));
|
||||
assert_eq!(loaded.pending_disks.len(), 1);
|
||||
assert!(disks[3].is_none(), "unformatted slot must not count as a member");
|
||||
let set = crate::set_disk::SetDisks::new_with_instance_ctx(
|
||||
"pending-format-test".to_string(),
|
||||
Arc::new(tokio::sync::RwLock::new(disks)),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
loaded.format,
|
||||
Vec::new(),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
PendingFormatFixture {
|
||||
ctx,
|
||||
set,
|
||||
pending: loaded.pending_disks.remove(0),
|
||||
receiver,
|
||||
format,
|
||||
_temp: temp,
|
||||
}
|
||||
}
|
||||
|
||||
async fn publish_pending_format(fixture: &PendingFormatFixture) {
|
||||
let bootstrap = crate::store::peer::BootstrapLocalTarget::new(fixture.ctx.clone());
|
||||
// Resolve the endpoint again, just as the writer's next RPC does. A
|
||||
// retained Arc alone would bypass the receiver quarantine regression.
|
||||
let resolved = bootstrap
|
||||
.local_mutation_disk(&fixture.receiver.endpoint().to_string())
|
||||
.await
|
||||
.expect("pending receiver must remain endpoint-addressable");
|
||||
assert!(Arc::ptr_eq(&resolved, &fixture.receiver));
|
||||
bootstrap
|
||||
.begin_legacy_local_rename_file(
|
||||
&fixture.receiver.endpoint().to_string(),
|
||||
(RUSTFS_META_BUCKET, "pending-format"),
|
||||
(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("writer's next rename should still be admitted")
|
||||
.commit()
|
||||
.await
|
||||
.expect("writer should complete the last format");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_preserves_pending_bootstrap_receiver_and_restores_completed_slot() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
publish_pending_format(&fixture).await;
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("completed format should reconcile before Ready");
|
||||
let slots = fixture.set.disks.read().await;
|
||||
assert_eq!(slots.iter().flatten().count(), 4);
|
||||
assert!(Arc::ptr_eq(
|
||||
slots[3].as_ref().expect("restored local member should exist"),
|
||||
&fixture.receiver
|
||||
));
|
||||
for (index, disk) in slots.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("all four members should be restored");
|
||||
let loaded = load_format_erasure(disk, false)
|
||||
.await
|
||||
.expect("restored member format should read");
|
||||
assert_eq!(loaded.shared_identity(), fixture.format.shared_identity());
|
||||
assert_eq!(loaded.erasure.this, fixture.format.erasure.sets[0][index]);
|
||||
}
|
||||
let id = fixture.format.erasure.sets[0][3];
|
||||
assert_eq!(
|
||||
slots[3]
|
||||
.as_ref()
|
||||
.expect("last member should exist")
|
||||
.get_disk_id()
|
||||
.await
|
||||
.expect("set disk ID should read"),
|
||||
Some(id)
|
||||
);
|
||||
assert_eq!(fixture.receiver.get_disk_id().await.expect("receiver ID should read"), Some(id));
|
||||
assert_eq!(
|
||||
fixture.ctx.local_disk_id_map().read().await.get(&id),
|
||||
Some(&fixture.receiver.endpoint().to_string())
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.ctx.local_disk_set_drives().read().await[0][0][3]
|
||||
.as_ref()
|
||||
.expect("local slot should remain active"),
|
||||
&fixture.receiver
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_quarantines_still_unformatted_receiver_before_ready() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("unformatted minority should be isolated without changing quorum");
|
||||
assert!(fixture.ctx.local_disk_map().read().await[&fixture.receiver.endpoint().to_string()].is_none());
|
||||
assert!(fixture.ctx.local_disk_set_drives().read().await[0][0][3].is_none());
|
||||
assert!(fixture.set.disks.read().await[3].is_none());
|
||||
assert_eq!(fixture.set.disks.read().await.iter().flatten().count(), 3);
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("a later init retry must preserve completed quarantine");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_quarantines_foreign_completed_receiver_before_ready() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][3];
|
||||
write_format_file(&fixture.receiver, &foreign)
|
||||
.await
|
||||
.expect("foreign format should publish");
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("foreign minority should be isolated");
|
||||
assert!(fixture.ctx.local_disk_map().read().await[&fixture.receiver.endpoint().to_string()].is_none());
|
||||
assert!(fixture.ctx.local_disk_set_drives().read().await[0][0][3].is_none());
|
||||
assert!(fixture.set.disks.read().await[3].is_none());
|
||||
assert!(
|
||||
!fixture
|
||||
.ctx
|
||||
.local_disk_id_map()
|
||||
.read()
|
||||
.await
|
||||
.contains_key(&foreign.erasure.this)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_rechecks_replaced_receiver_after_io_and_accepts_current_member_on_retry() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
publish_pending_format(&fixture).await;
|
||||
let observed = load_format_erasure(&fixture.receiver, false)
|
||||
.await
|
||||
.expect("old receiver observation should load");
|
||||
let replacement = new_disk(
|
||||
&fixture.receiver.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement receiver should open");
|
||||
fixture
|
||||
.ctx
|
||||
.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(replacement.endpoint().to_string(), Some(replacement.clone()));
|
||||
fixture.ctx.local_disk_set_drives().write().await[0][0][3] = Some(replacement.clone());
|
||||
*fixture.set.disks.write().await.get_mut(3).expect("last slot should exist") = Some(replacement.clone());
|
||||
assert!(
|
||||
fixture
|
||||
.pending
|
||||
.publish_observation(&fixture.ctx, &fixture.set, None, Some(&fixture.receiver), Ok(observed))
|
||||
.await
|
||||
.is_err(),
|
||||
"stale I/O must not publish over a replacement"
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.ctx.local_disk_map().read().await[&replacement.endpoint().to_string()]
|
||||
.as_ref()
|
||||
.expect("replacement must not be quarantined"),
|
||||
&replacement
|
||||
));
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("existing init retry should validate the current replacement");
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.set.disks.read().await[3]
|
||||
.as_ref()
|
||||
.expect("validated slot should exist"),
|
||||
&replacement
|
||||
));
|
||||
assert_eq!(
|
||||
replacement.get_disk_id().await.expect("replacement ID should read"),
|
||||
Some(fixture.format.erasure.sets[0][3])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_remote_pending_publication_checks_shared_identity_and_slot() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let format = FormatV3::new(1, 4);
|
||||
let mut endpoint = Endpoint::try_from("http://remote-server:9000/data").expect("remote endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(3);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remote client should initialize without connecting");
|
||||
let mut endpoints = vec![endpoint.clone(); 4];
|
||||
for (index, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_disk_index(index);
|
||||
}
|
||||
let set = crate::set_disk::SetDisks::new_with_instance_ctx(
|
||||
"remote-pending-test".to_string(),
|
||||
Arc::new(tokio::sync::RwLock::new(vec![None; 4])),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
format.clone(),
|
||||
Vec::new(),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
let pending = PendingFormatDisk { disk: disk.clone() };
|
||||
let mut wrong_slot = format.clone();
|
||||
wrong_slot.erasure.this = format.erasure.sets[0][2];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(wrong_slot))
|
||||
.await
|
||||
.expect("wrong-slot format should remain excluded");
|
||||
assert!(set.disks.read().await[3].is_none());
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][3];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(foreign))
|
||||
.await
|
||||
.expect("foreign remote format should remain excluded");
|
||||
assert!(set.disks.read().await[3].is_none());
|
||||
let mut matching = format.clone();
|
||||
matching.erasure.this = format.erasure.sets[0][3];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(matching))
|
||||
.await
|
||||
.expect("exact remote slot should restore");
|
||||
assert!(Arc::ptr_eq(set.disks.read().await[3].as_ref().expect("remote slot should exist"), &disk));
|
||||
assert_eq!(
|
||||
disk.get_disk_id().await.expect("remote ID state should read"),
|
||||
Some(format.erasure.sets[0][3])
|
||||
);
|
||||
assert!(
|
||||
ctx.local_disk_map().read().await.is_empty(),
|
||||
"remote restoration must not register a local receiver"
|
||||
);
|
||||
}
|
||||
|
||||
async fn two_local_disks_with_missing_third() -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
|
||||
let (temp_dir, mut disks) = local_disks(2).await;
|
||||
disks.push(None);
|
||||
|
||||
@@ -431,8 +431,8 @@ pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
pub use bucket_fence::BucketIncarnationFenceGuard;
|
||||
pub(crate) mod delete_recovery_runtime;
|
||||
mod heal;
|
||||
pub(crate) use heal::{BucketHealScope, bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
mod init;
|
||||
@@ -494,6 +494,7 @@ pub struct ECStore {
|
||||
/// Memoizes bucket-incarnation validation under continuous lifecycle
|
||||
/// read-lock coverage (see [`bucket_fence`]).
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
pub(crate) ordinary_classification: crate::bucket::artifact::OrdinaryClassificationRegistry,
|
||||
}
|
||||
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
|
||||
@@ -1430,7 +1431,7 @@ impl BucketOperations for ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||
Box::pin(self.handle_make_bucket(bucket, opts)).await
|
||||
self.handle_make_bucket(bucket, opts).await
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -1904,6 +1905,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
};
|
||||
|
||||
let rendered = format!("{store:?}");
|
||||
@@ -1919,6 +1921,19 @@ mod tests {
|
||||
assert!(!rendered.contains("DiskHealthTracker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_bucket_handler_future_remains_stack_bounded() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let opts = MakeBucketOptions::default();
|
||||
let future = store.handle_make_bucket("stack-bounded-bucket", &opts);
|
||||
let future_size = std::mem::size_of_val(&future);
|
||||
|
||||
assert!(
|
||||
future_size <= 4 * 1024,
|
||||
"make bucket handler future must remain stack-bounded; measured {future_size} bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_options_debug_does_not_expand_tier_store_handle() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
@@ -1979,6 +1994,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -602,7 +602,7 @@ impl ECStore {
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||
check_new_object_args(bucket, object, opts.data_movement)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
|
||||
if self.single_pool() {
|
||||
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
|
||||
@@ -741,10 +741,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -778,7 +775,7 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart upload requires data_movement options"));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
let pool = self.pools.get(target_pool_idx).ok_or_else(|| {
|
||||
Error::InvalidArgument("data-movement".to_string(), "target-pool".to_string(), target_pool_idx.to_string())
|
||||
@@ -1160,7 +1157,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -1261,7 +1258,7 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
|
||||
let expected_incarnation_id = opts
|
||||
@@ -1666,6 +1663,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+600
-453
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
# Compression-index compatibility fixtures
|
||||
|
||||
These are synthetic public fixtures, generated with RustFS source and the locked dependencies at `017b85a5bfc3c2c779af942b68e3a7c42b3a9d81`. They contain no captured user object or credentials.
|
||||
|
||||
`legacy-lz4-stream.bin` was emitted by `rustfs_rio::CompressReader` using LZ4 and its default 1 MiB block size. The compressor and utility compression source files are byte-identical between this commit and `17aa75a1759e856e93b8c1df2fd447b5ce92e30d`.
|
||||
|
||||
`pre-zigzag-index.bin` was emitted by compiling the complete, unmodified `crates/rio/src/compress_index.rs` module from `17aa75a1759e856e93b8c1df2fd447b5ce92e30d`, before the signed-varint fix. The compressor's serializable Index state was deserialized into that historical Index type and passed to its writer. Its historical reader verified complete consumption, identical state and two seek results before the bytes were retained. This pins a historical writer module; it is not a full historical release or a captured stored object.
|
||||
|
||||
`zigzag-index.bin` was emitted from the same Index state using the current writer, and serves as an indexed-read control. The old and new index files intentionally differ.
|
||||
|
||||
## Reproducing the input
|
||||
|
||||
The plaintext length is `3 * 1024 * 1024 + 123`. For each byte offset, let `chunk = offset / 1024` and `lane = offset % 1024`. The first eight bytes of each 1024-byte chunk are `u64::to_le_bytes(chunk)`; the remainder is `b"abcdefghijklmnopqrstuvwxyz"[(offset + chunk) % 26]`.
|
||||
|
||||
Drain the legacy LZ4 CompressReader into a byte vector, obtain its Index through TryGetIndex, then serialize that Index state through serde_json. Compile the pinned historical module, deserialize the state into its Index, and call into_vec. Emit the current writer's into_vec result separately. Do not recreate the old index through the reader being tested.
|
||||
|
||||
The recorded state has `est_block_uncomp = 0`, total compressed offset `52137`, total uncompressed offset `3145728`, and these `(compressed, uncompressed)` entries:
|
||||
|
||||
- `(17085, 1048576)`
|
||||
- `(34663, 2097152)`
|
||||
- `(52137, 3145728)`
|
||||
|
||||
The final partial block is part of the stream; the recorded totals are the Index state exposed by this historical compressor, not a replacement for the object's actual length.
|
||||
|
||||
| Bytes | Length | SHA-256 |
|
||||
| --- | ---: | --- |
|
||||
| Plaintext | 3145851 | `68f9edf6f9600395d6d2b5517d97570fecd2a0c6dbe6130617ef415a3b48d89c` |
|
||||
| Legacy LZ4 stream | 52212 | `cbb8ead867a159a6a089a3a09fd424986cc8995bcb384434d0b95d0d9bdfef7e` |
|
||||
| Historical index | 47 | `ff4fb74d14a8ddc0d3c31d565c30b62e142df02ac0d19fa270c666b98a8e7171` |
|
||||
| Current index | 51 | `8a819221f9bdaa2bd05e14eead2833746ea01cb1c68086bffa25a731c252174e` |
|
||||
|
||||
Tests exercise the production read planner and GetObjectReader rather than decoding the expected bytes with a replacement implementation. Historical-body and fallback cases apply to both legacy and rio-v2 builds; the exact legacy indexed-offset control is feature-gated because each backend has its own index representation. No throughput or application-level release qualification is inferred from these fixtures.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -214,6 +214,9 @@ const MRF_LIFECYCLE_MAGIC: &[u8; 8] = b"RFMFLC01";
|
||||
const MRF_LIFECYCLE_FORMAT_VERSION: u8 = 1;
|
||||
const MRF_LIFECYCLE_FORMAT_VERSION_V2: u8 = 2;
|
||||
|
||||
/// Highest supported lifecycle format; every format from 1 through this value is readable.
|
||||
pub const MRF_LIFECYCLE_READER_MAX_FORMAT_VERSION: u8 = MRF_LIFECYCLE_FORMAT_VERSION_V2;
|
||||
|
||||
/// Record format tag.
|
||||
const MRF_JOURNAL_FORMAT: u8 = 1;
|
||||
/// Record layout version.
|
||||
|
||||
@@ -61,6 +61,20 @@ fn lifecycle_v2_reader_accepts_retained_unknown_generation_without_a_source_iden
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_reader_capability_matches_supported_formats() {
|
||||
let mut payload = unknown_generation_payload();
|
||||
payload["records"][0]["state"] = json!({"state": "active"});
|
||||
for version in 0..=MRF_LIFECYCLE_READER_MAX_FORMAT_VERSION + 1 {
|
||||
payload["format_version"] = json!(version);
|
||||
assert_eq!(
|
||||
decode_mrf_lifecycle_checkpoint(&lifecycle_reader_frame(&payload, version), 8192).is_ok(),
|
||||
(1..=MRF_LIFECYCLE_READER_MAX_FORMAT_VERSION).contains(&version),
|
||||
"advertised reader support must match the actual decoder for format {version}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_v2_reader_rejects_mislabeled_and_unsupported_formats() {
|
||||
let payload = unknown_generation_payload();
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Durable restrictions owned by an authenticated user. A record is not an IAM
|
||||
//! grant or credential; issuing credentials still requires current IAM checks.
|
||||
use crate::{error::is_err_config_not_found, store::Store};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const PREFIX: &str = "config/iam/plugin-delegations/v1";
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Restrictions {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
|
||||
pub version: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
|
||||
pub output_bucket: Option<String>,
|
||||
pub deployment: String,
|
||||
pub bucket: String,
|
||||
pub input_prefix: String,
|
||||
pub output_prefix: String,
|
||||
pub expires_at_unix_seconds: u64,
|
||||
}
|
||||
|
||||
// An absent v1 field is compatible; an explicitly null v2 field is invalid.
|
||||
fn present_value<'de, T: Deserialize<'de>, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Option<T>, D::Error> {
|
||||
T::deserialize(deserializer).map(Some)
|
||||
}
|
||||
|
||||
impl Restrictions {
|
||||
pub fn output_bucket(&self) -> Result<&str, DelegationError> {
|
||||
match (self.version, self.output_bucket.as_deref()) {
|
||||
(None, None) => Ok(&self.bucket),
|
||||
(Some(2), Some(output)) => Ok(output),
|
||||
_ => Err(DelegationError::Invalid),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self, now: u64) -> Result<(), DelegationError> {
|
||||
let output_bucket = self.output_bucket()?;
|
||||
let names: [&str; 5] = [
|
||||
&self.deployment,
|
||||
&self.bucket,
|
||||
&self.input_prefix,
|
||||
&self.output_prefix,
|
||||
output_bucket,
|
||||
];
|
||||
if names
|
||||
.iter()
|
||||
.any(|s| s.is_empty() || s.len() > 1024 || s.contains("${") || s.chars().any(char::is_control))
|
||||
|| self.bucket.contains('/')
|
||||
|| self.bucket.contains('*')
|
||||
|| self.bucket.contains('?')
|
||||
|| output_bucket.contains('/')
|
||||
|| output_bucket.contains('*')
|
||||
|| output_bucket.contains('?')
|
||||
|| self.input_prefix.contains('*')
|
||||
|| self.input_prefix.contains('?')
|
||||
|| self.output_prefix.contains('*')
|
||||
|| self.output_prefix.contains('?')
|
||||
|| self.expires_at_unix_seconds <= now
|
||||
|| self.expires_at_unix_seconds.saturating_sub(now) > 86400
|
||||
{
|
||||
return Err(DelegationError::Invalid);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Delegation {
|
||||
pub version: u32,
|
||||
pub id: String,
|
||||
pub revision: String,
|
||||
pub owner_access_key: String,
|
||||
pub created_at_unix_seconds: u64,
|
||||
pub restrictions: Restrictions,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Tombstone {
|
||||
version: u32,
|
||||
id: String,
|
||||
revision: String,
|
||||
owner_access_key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum DelegationError {
|
||||
#[error("invalid delegation restriction")]
|
||||
Invalid,
|
||||
#[error("delegation not found")]
|
||||
NotFound,
|
||||
#[error("delegation owner rejected")]
|
||||
OwnerRejected,
|
||||
#[error("delegation unavailable")]
|
||||
Unavailable,
|
||||
#[error("delegation revoked")]
|
||||
Revoked,
|
||||
#[error("delegation expired")]
|
||||
Expired,
|
||||
}
|
||||
|
||||
fn record_path(id: &str) -> Result<String, DelegationError> {
|
||||
if id.len() != 32 || !id.bytes().all(|c| c.is_ascii_hexdigit()) {
|
||||
return Err(DelegationError::Invalid);
|
||||
}
|
||||
Ok(format!("{PREFIX}/{id}/record.json"))
|
||||
}
|
||||
|
||||
async fn load<T: Store>(store: &T, id: &str) -> Result<Delegation, DelegationError> {
|
||||
let record: Delegation = store.load_iam_config_strict(record_path(id)?).await.map_err(|error| {
|
||||
if is_err_config_not_found(&error) {
|
||||
DelegationError::NotFound
|
||||
} else {
|
||||
DelegationError::Unavailable
|
||||
}
|
||||
})?;
|
||||
if !matches!(record.version, 1 | 2)
|
||||
|| record.version != record.restrictions.version.unwrap_or(1)
|
||||
|| record.id != id
|
||||
|| record.revision != id
|
||||
|| record.owner_access_key.is_empty()
|
||||
{
|
||||
return Err(DelegationError::Unavailable);
|
||||
}
|
||||
record
|
||||
.restrictions
|
||||
.validate(record.created_at_unix_seconds)
|
||||
.map_err(|_| DelegationError::Unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
pub(crate) async fn create<T: Store>(
|
||||
store: &T,
|
||||
owner: &str,
|
||||
restrictions: Restrictions,
|
||||
now: u64,
|
||||
) -> Result<Delegation, DelegationError> {
|
||||
restrictions.validate(now)?;
|
||||
if owner.is_empty() {
|
||||
return Err(DelegationError::OwnerRejected);
|
||||
}
|
||||
// IDs are generated exclusively here; no update operation accepts an ID.
|
||||
let id = format!("{:032x}", rand::random::<u128>());
|
||||
let path = record_path(&id)?;
|
||||
match store.load_iam_config_strict::<Delegation>(&path).await {
|
||||
Err(error) if is_err_config_not_found(&error) => {}
|
||||
_ => return Err(DelegationError::Unavailable),
|
||||
}
|
||||
let record = Delegation {
|
||||
version: restrictions.version.unwrap_or(1),
|
||||
revision: id.clone(),
|
||||
id,
|
||||
owner_access_key: owner.to_owned(),
|
||||
created_at_unix_seconds: now,
|
||||
restrictions,
|
||||
};
|
||||
store
|
||||
.save_iam_config(&record, path)
|
||||
.await
|
||||
.map_err(|_| DelegationError::Unavailable)?;
|
||||
if load(store, &record.id).await? != record {
|
||||
return Err(DelegationError::Unavailable);
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
pub(crate) async fn revoke<T: Store>(store: &T, owner: &str, id: &str) -> Result<(), DelegationError> {
|
||||
let record = load(store, id).await?;
|
||||
if record.owner_access_key != owner {
|
||||
return Err(DelegationError::OwnerRejected);
|
||||
}
|
||||
let tombstone = Tombstone {
|
||||
version: 1,
|
||||
id: record.id,
|
||||
revision: record.revision,
|
||||
owner_access_key: record.owner_access_key,
|
||||
};
|
||||
let path = format!("{PREFIX}/{id}/revoked.json");
|
||||
// Every retry writes exactly the same revocation; active records never change.
|
||||
store
|
||||
.save_iam_config(&tombstone, &path)
|
||||
.await
|
||||
.map_err(|_| DelegationError::Unavailable)?;
|
||||
let saved: Tombstone = store
|
||||
.load_iam_config_strict(path)
|
||||
.await
|
||||
.map_err(|_| DelegationError::Unavailable)?;
|
||||
if saved != tombstone {
|
||||
return Err(DelegationError::Unavailable);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn read_active<T: Store>(store: &T, owner: &str, id: &str, now: u64) -> Result<Delegation, DelegationError> {
|
||||
let record = load(store, id).await?;
|
||||
if record.owner_access_key != owner {
|
||||
return Err(DelegationError::OwnerRejected);
|
||||
}
|
||||
check_active(store, record, now).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_owned<T: Store>(store: &T, owner: &str, id: &str) -> Result<Delegation, DelegationError> {
|
||||
let record = load(store, id).await?;
|
||||
if record.owner_access_key != owner {
|
||||
return Err(DelegationError::OwnerRejected);
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_active_for_broker<T: Store>(store: &T, id: &str, now: u64) -> Result<Delegation, DelegationError> {
|
||||
let record = load(store, id).await?;
|
||||
check_active(store, record, now).await
|
||||
}
|
||||
async fn check_active<T: Store>(store: &T, record: Delegation, now: u64) -> Result<Delegation, DelegationError> {
|
||||
let id = &record.id;
|
||||
match store
|
||||
.load_iam_config_strict::<Tombstone>(format!("{PREFIX}/{id}/revoked.json"))
|
||||
.await
|
||||
{
|
||||
Ok(_) => return Err(DelegationError::Revoked),
|
||||
Err(error) if is_err_config_not_found(&error) => {}
|
||||
Err(_) => return Err(DelegationError::Unavailable),
|
||||
}
|
||||
if now >= record.restrictions.expires_at_unix_seconds {
|
||||
return Err(DelegationError::Expired);
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn delegation_v1_json_remains_compatible() {
|
||||
let legacy = serde_json::json!({"deployment":"runner-a","bucket":"photos",
|
||||
"input_prefix":"inputs/","output_prefix":"outputs/","expires_at_unix_seconds":200});
|
||||
let restriction: Restrictions = serde_json::from_value(legacy.clone()).expect("legacy restriction");
|
||||
restriction.validate(100).expect("legacy validation");
|
||||
assert_eq!(restriction.output_bucket().expect("v1 output"), "photos");
|
||||
assert_eq!(serde_json::to_value(restriction).expect("legacy serialization"), legacy);
|
||||
let record = serde_json::json!({"version":1,"id":"a".repeat(32),"revision":"a".repeat(32),
|
||||
"owner_access_key":"owner","created_at_unix_seconds":100,"restrictions":legacy});
|
||||
let decoded: Delegation = serde_json::from_value(record.clone()).expect("legacy record");
|
||||
assert_eq!(serde_json::to_value(decoded).expect("legacy record serialization"), record);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delegation_v2_requires_explicit_output_bucket() {
|
||||
let valid = serde_json::json!({"version":2,"deployment":"runner-a","bucket":"photos",
|
||||
"input_prefix":"inputs/","output_bucket":"artifacts","output_prefix":"outputs/",
|
||||
"expires_at_unix_seconds":200});
|
||||
let restriction: Restrictions = serde_json::from_value(valid.clone()).expect("v2 restriction");
|
||||
restriction.validate(100).expect("explicit output bucket");
|
||||
assert_eq!(restriction.output_bucket().expect("v2 output"), "artifacts");
|
||||
assert_eq!(serde_json::to_value(restriction).expect("v2 serialization"), valid);
|
||||
for key in ["version", "output_bucket"] {
|
||||
let mut bad = valid.clone();
|
||||
bad.as_object_mut().expect("object").remove(key);
|
||||
assert!(
|
||||
serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
|
||||
"v2 required {key}"
|
||||
);
|
||||
}
|
||||
for (key, value) in [
|
||||
("version", serde_json::json!(1)),
|
||||
("version", serde_json::json!(3)),
|
||||
("version", serde_json::Value::Null),
|
||||
("output_bucket", serde_json::Value::Null),
|
||||
("output_bucket", serde_json::json!("")),
|
||||
("output_bucket", serde_json::json!("other/*")),
|
||||
("output_bucket", serde_json::json!("${aws:username}")),
|
||||
("output_bucket", serde_json::json!("other?")),
|
||||
("output_bucket", serde_json::json!("other\n")),
|
||||
("unknown", serde_json::json!(true)),
|
||||
] {
|
||||
let mut bad = valid.clone();
|
||||
bad[key] = value;
|
||||
assert!(
|
||||
serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
|
||||
"invalid v2 {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restrictions_are_literal_bounded_and_expiring() {
|
||||
let mut restriction = Restrictions {
|
||||
version: None,
|
||||
output_bucket: None,
|
||||
deployment: "runner-a".into(),
|
||||
bucket: "photos".into(),
|
||||
input_prefix: "inputs/".into(),
|
||||
output_prefix: "outputs/".into(),
|
||||
expires_at_unix_seconds: 200,
|
||||
};
|
||||
assert!(restriction.validate(100).is_ok());
|
||||
restriction.output_prefix = "outputs/*".into();
|
||||
assert!(matches!(restriction.validate(100), Err(DelegationError::Invalid)));
|
||||
restriction.output_prefix = "outputs/".into();
|
||||
assert!(matches!(restriction.validate(200), Err(DelegationError::Invalid)));
|
||||
restriction.expires_at_unix_seconds = 200;
|
||||
restriction.input_prefix = "inputs/${aws:username}/".into();
|
||||
assert!(
|
||||
matches!(restriction.validate(100), Err(DelegationError::Invalid)),
|
||||
"C07_LITERAL_SCOPE_BOUND: policy variables are not literal restrictions"
|
||||
);
|
||||
assert!(record_path("../../users").is_err());
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ pub(crate) const EVENT_OIDC_DIAGNOSTICS: &str = "oidc_diagnostics";
|
||||
const EVENT_OIDC_STATE: &str = "oidc_state";
|
||||
|
||||
pub mod cache;
|
||||
pub mod delegation;
|
||||
pub mod error;
|
||||
pub mod federation;
|
||||
pub mod keyring;
|
||||
|
||||
+30
-42
@@ -3409,27 +3409,23 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_jwt_claims_basic() {
|
||||
let user_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: Credentials {
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
session_token: "invalid-token".to_string(), // Invalid token for testing error handling
|
||||
expiration: None,
|
||||
status: "enabled".to_string(),
|
||||
parent_user: "".to_string(),
|
||||
groups: None,
|
||||
claims: Some({
|
||||
let mut claims = HashMap::new();
|
||||
claims.insert("sub".to_string(), json!("test-user"));
|
||||
claims.insert("aud".to_string(), json!("test-audience"));
|
||||
claims
|
||||
}),
|
||||
name: None,
|
||||
description: None,
|
||||
},
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let user_identity = UserIdentity::new(Credentials {
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
session_token: "invalid-token".to_string(), // Invalid token for testing error handling
|
||||
expiration: None,
|
||||
status: "enabled".to_string(),
|
||||
parent_user: "".to_string(),
|
||||
groups: None,
|
||||
claims: Some({
|
||||
let mut claims = HashMap::new();
|
||||
claims.insert("sub".to_string(), json!("test-user"));
|
||||
claims.insert("aud".to_string(), json!("test-audience"));
|
||||
claims
|
||||
}),
|
||||
name: None,
|
||||
description: None,
|
||||
});
|
||||
|
||||
let result = extract_jwt_claims(&user_identity);
|
||||
// In test environment without proper JWT setup, this should fail
|
||||
@@ -3438,22 +3434,18 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_jwt_claims_no_claims() {
|
||||
let user_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: Credentials {
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
session_token: "".to_string(), // Empty token
|
||||
expiration: None,
|
||||
status: "enabled".to_string(),
|
||||
parent_user: "".to_string(),
|
||||
groups: None,
|
||||
claims: None,
|
||||
name: None,
|
||||
description: None,
|
||||
},
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let user_identity = UserIdentity::new(Credentials {
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
session_token: "".to_string(), // Empty token
|
||||
expiration: None,
|
||||
status: "enabled".to_string(),
|
||||
parent_user: "".to_string(),
|
||||
groups: None,
|
||||
claims: None,
|
||||
name: None,
|
||||
description: None,
|
||||
});
|
||||
|
||||
let result = extract_jwt_claims(&user_identity);
|
||||
// Should fail with empty session token
|
||||
@@ -3512,11 +3504,7 @@ mod tests {
|
||||
description: None,
|
||||
};
|
||||
|
||||
let user_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials,
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let user_identity = UserIdentity::new(credentials);
|
||||
|
||||
// Test basic structure
|
||||
assert_eq!(user_identity.version, 1);
|
||||
|
||||
@@ -91,6 +91,7 @@ pub(crate) mod crate_boundary {
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
|
||||
}
|
||||
|
||||
pub(crate) mod runtime {
|
||||
|
||||
@@ -26,6 +26,11 @@ pub trait Store: Clone + Send + Sync + 'static {
|
||||
fn has_watcher(&self) -> bool;
|
||||
async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()>;
|
||||
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item>;
|
||||
/// Only confirmed storage absence may return ConfigNotFound. Corrupt or
|
||||
/// undecryptable objects must remain errors, especially for revocations.
|
||||
async fn load_iam_config_strict<Item: DeserializeOwned>(&self, _path: impl AsRef<str> + Send) -> Result<Item> {
|
||||
Err(crate::error::Error::IamSysNotInitialized)
|
||||
}
|
||||
async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
|
||||
|
||||
async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;
|
||||
|
||||
@@ -42,6 +42,7 @@ use tracing::{debug, error, warn};
|
||||
|
||||
use crate::storage_api::object_store::{
|
||||
HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
read_config_preserve_empty_with_metadata_opts,
|
||||
};
|
||||
|
||||
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
|
||||
@@ -414,9 +415,14 @@ impl ObjectStore {
|
||||
&self,
|
||||
path: impl AsRef<str> + Send,
|
||||
mode: LoadMode,
|
||||
preserve_empty: bool,
|
||||
) -> Result<(Vec<u8>, IamObjectInfo)> {
|
||||
let path_ref = path.as_ref();
|
||||
let (data, obj) = read_iam_config_with_metadata(self.object_api.clone(), path_ref, &mode.read_opts()).await?;
|
||||
let (data, obj) = if preserve_empty {
|
||||
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path_ref, &mode.read_opts()).await?
|
||||
} else {
|
||||
read_iam_config_with_metadata(self.object_api.clone(), path_ref, &mode.read_opts()).await?
|
||||
};
|
||||
let outcome = Self::decrypt_data_with_source(&data)?;
|
||||
self.maybe_schedule_lazy_rewrite(path_ref, &outcome, &obj);
|
||||
|
||||
@@ -611,11 +617,17 @@ impl ObjectStore {
|
||||
Ok(serde_json::from_slice(&outcome.plain)?)
|
||||
}
|
||||
|
||||
fn decode_iam_config_strict<Item: DeserializeOwned>(data: &[u8]) -> Result<Item> {
|
||||
let outcome = Self::decrypt_data_with_source(data)?;
|
||||
Ok(serde_json::from_slice(&outcome.plain)?)
|
||||
}
|
||||
|
||||
/// Parameterized core of [`Store::load_user_identity`].
|
||||
async fn load_user_identity_with(&self, name: &str, user_type: UserType, mode: LoadMode) -> Result<UserIdentity> {
|
||||
let mut u: UserIdentity = self
|
||||
.load_iam_config_with(get_user_identity_path(name, user_type), mode)
|
||||
.load_iamconfig_bytes_with_metadata(get_user_identity_path(name, user_type), mode, true)
|
||||
.await
|
||||
.and_then(|(data, _)| serde_json::from_slice(&data).map_err(Error::from))
|
||||
.map_err(|err| {
|
||||
if is_err_config_not_found(&err) {
|
||||
debug!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity missing");
|
||||
@@ -695,7 +707,7 @@ impl ObjectStore {
|
||||
/// Parameterized core of [`Store::load_policy`].
|
||||
async fn load_policy_with(&self, name: &str, mode: LoadMode) -> Result<PolicyDoc> {
|
||||
let (data, obj) = self
|
||||
.load_iamconfig_bytes_with_metadata(get_policy_doc_path(name), mode)
|
||||
.load_iamconfig_bytes_with_metadata(get_policy_doc_path(name), mode, false)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if is_err_config_not_found(&err) {
|
||||
@@ -824,6 +836,13 @@ impl Store for ObjectStore {
|
||||
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
|
||||
self.load_iam_config_with(path, LoadMode::Locked).await
|
||||
}
|
||||
async fn load_iam_config_strict<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
|
||||
let (data, _) =
|
||||
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path.as_ref(), &LoadMode::Locked.read_opts())
|
||||
.await?;
|
||||
Self::decode_iam_config_strict(&data)
|
||||
}
|
||||
|
||||
/// Saves IAM configuration with a retry mechanism on failure.
|
||||
///
|
||||
/// Attempts to save the IAM configuration up to 5 times if the storage layer is not ready,
|
||||
@@ -1458,6 +1477,40 @@ mod tests {
|
||||
assert!(result.is_err(), "corrupt stream_io data should fail decrypt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_revocation_load_keeps_real_crypto_failure_distinct_from_absence() {
|
||||
let cred = test_cred();
|
||||
let key = format!("{}:{}", cred.access_key, cred.secret_key);
|
||||
let plain = br#"{"version":1,"id":"revoked"}"#;
|
||||
let mut encrypted = rustfs_crypto::encrypt_stream_io(key.as_bytes(), plain).expect("real stored ciphertext");
|
||||
// Exercise the exact decoder used after the actual object read.
|
||||
assert!(ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted).is_ok());
|
||||
encrypted[50] ^= 0xFF;
|
||||
let error = ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted)
|
||||
.expect_err("corrupt tombstone must not become missing");
|
||||
assert!(
|
||||
matches!(error, Error::CryptoError(_)),
|
||||
"C07_REVOCATION_FAIL_CLOSED: preserve actual crypto error"
|
||||
);
|
||||
assert!(
|
||||
!crate::error::is_err_config_not_found(&error),
|
||||
"C07_REVOCATION_FAIL_CLOSED: corrupted revocation is not absence"
|
||||
);
|
||||
// The preserve-empty reader keeps an existing zero-byte object as
|
||||
// Ok(empty); ecstore's read_config_preserve_empty_distinguishes_empty_object
|
||||
// covers that storage boundary. Its payload must fail here, not become
|
||||
// ConfigNotFound and reactivate a revoked delegation.
|
||||
let empty =
|
||||
ObjectStore::decode_iam_config_strict::<serde_json::Value>(&[]).expect_err("an existing empty tombstone is corrupt");
|
||||
assert!(
|
||||
!crate::error::is_err_config_not_found(&empty),
|
||||
"C07_REVOCATION_FAIL_CLOSED: empty existing revocation is not absence"
|
||||
);
|
||||
let invalid_json =
|
||||
ObjectStore::decode_iam_config_strict::<serde_json::Value>(br#"{"version": }"#).expect_err("invalid stored JSON");
|
||||
assert!(!crate::error::is_err_config_not_found(&invalid_json));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decrypt_data_short_data_fails() {
|
||||
let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON
|
||||
|
||||
+265
-27
@@ -75,6 +75,7 @@ pub const STATUS_ENABLED: &str = "enabled";
|
||||
pub const STATUS_DISABLED: &str = "disabled";
|
||||
|
||||
pub const POLICYNAME: &str = "policy";
|
||||
pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
|
||||
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
|
||||
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
|
||||
const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-policy";
|
||||
@@ -168,6 +169,12 @@ enum PreparedServicePolicyMode {
|
||||
|
||||
#[derive(Clone)]
|
||||
enum PreparedIamMode {
|
||||
TerminalDeny,
|
||||
PluginAttemptSts {
|
||||
is_owner: bool,
|
||||
combined_policy: Policy,
|
||||
session_policy: PreparedSessionPolicy,
|
||||
},
|
||||
Opa,
|
||||
Owner,
|
||||
Deny,
|
||||
@@ -202,7 +209,7 @@ impl PreparedIamAuth {
|
||||
pub async fn needs_existing_object_tag_for_args(&self, args: &Args<'_>) -> bool {
|
||||
match &self.mode {
|
||||
PreparedIamMode::Opa => true,
|
||||
PreparedIamMode::Owner | PreparedIamMode::Deny => false,
|
||||
PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => false,
|
||||
PreparedIamMode::Regular { combined_policy } => {
|
||||
policy_needs_existing_object_tag_for_args(combined_policy, args).await
|
||||
}
|
||||
@@ -210,6 +217,11 @@ impl PreparedIamAuth {
|
||||
combined_policy,
|
||||
session_policy,
|
||||
..
|
||||
}
|
||||
| PreparedIamMode::PluginAttemptSts {
|
||||
combined_policy,
|
||||
session_policy,
|
||||
..
|
||||
} => {
|
||||
policy_needs_existing_object_tag_for_args(combined_policy, args).await
|
||||
|| prepared_session_policy_needs_existing_object_tag_for_args(session_policy, args).await
|
||||
@@ -239,14 +251,56 @@ impl PreparedIamAuth {
|
||||
pub fn combined_policy_for_view(&self) -> Option<&Policy> {
|
||||
match &self.mode {
|
||||
PreparedIamMode::Regular { combined_policy } => Some(combined_policy),
|
||||
PreparedIamMode::Sts { combined_policy, .. } => Some(combined_policy),
|
||||
PreparedIamMode::Sts { combined_policy, .. } | PreparedIamMode::PluginAttemptSts { combined_policy, .. } => {
|
||||
Some(combined_policy)
|
||||
}
|
||||
PreparedIamMode::ServiceAccount { combined_policy, .. } => Some(combined_policy),
|
||||
PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny => None,
|
||||
PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Store> IamSys<T> {
|
||||
/// For the authenticated, explicitly configured broker handler only.
|
||||
pub async fn read_active_plugin_delegation_for_broker(
|
||||
&self,
|
||||
id: &str,
|
||||
now: u64,
|
||||
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
|
||||
crate::delegation::read_active_for_broker(&self.store.api, id, now).await
|
||||
}
|
||||
pub async fn create_plugin_delegation(
|
||||
&self,
|
||||
owner: &str,
|
||||
restrictions: crate::delegation::Restrictions,
|
||||
now: u64,
|
||||
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
|
||||
crate::delegation::create(&self.store.api, owner, restrictions, now).await
|
||||
}
|
||||
pub async fn revoke_plugin_delegation(
|
||||
&self,
|
||||
owner: &str,
|
||||
id: &str,
|
||||
) -> std::result::Result<(), crate::delegation::DelegationError> {
|
||||
crate::delegation::revoke(&self.store.api, owner, id).await
|
||||
}
|
||||
pub async fn read_active_plugin_delegation(
|
||||
&self,
|
||||
owner: &str,
|
||||
id: &str,
|
||||
now: u64,
|
||||
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
|
||||
crate::delegation::read_active(&self.store.api, owner, id, now).await
|
||||
}
|
||||
/// Reads an owned delegation record for historical lookups, including expired or revoked records.
|
||||
pub async fn read_owned_plugin_delegation(
|
||||
&self,
|
||||
owner: &str,
|
||||
id: &str,
|
||||
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
|
||||
crate::delegation::read_owned(&self.store.api, owner, id).await
|
||||
}
|
||||
|
||||
/// Create a new IamSys instance with the given IamCache store
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -281,6 +335,16 @@ impl<T: Store> IamSys<T> {
|
||||
*guard = PolicyPluginState::Ready(client);
|
||||
}
|
||||
|
||||
/// Restricted credentials require the built-in parent/session intersection.
|
||||
/// External authorization bypasses that path, so only Disabled is supported.
|
||||
pub async fn ensure_restricted_credential_vending() -> Result<()> {
|
||||
if matches!(Self::policy_plugin_state().await, PolicyPluginState::Disabled) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::IAMActionNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
|
||||
let policy_plugin_state = get_policy_plugin_state();
|
||||
let guard = policy_plugin_state.read().await;
|
||||
@@ -1383,17 +1447,50 @@ impl<T: Store> IamSys<T> {
|
||||
/// 1) know whether policy evaluation may need `s3:ExistingObjectTag`, and
|
||||
/// 2) evaluate with final conditions without re-merging identity policies.
|
||||
pub async fn prepare_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
|
||||
if args.is_owner {
|
||||
let marked = args.claims.contains_key(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
|
||||
if args.is_owner && !marked {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Owner,
|
||||
};
|
||||
}
|
||||
|
||||
if let Some(prepared) = Self::policy_plugin_state().await.prepared_iam_auth() {
|
||||
// One observed state decides this request's path. Marked credentials
|
||||
// never enter OPA, including after a restart or on another node.
|
||||
let state = Self::policy_plugin_state().await;
|
||||
if marked {
|
||||
if args.is_owner
|
||||
|| args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
|
||||
|| !matches!(state, PolicyPluginState::Disabled)
|
||||
{
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
}
|
||||
let prepared = self.prepare_builtin_auth(args).await;
|
||||
let mode = match prepared.mode {
|
||||
PreparedIamMode::Sts {
|
||||
is_owner,
|
||||
combined_policy,
|
||||
session_policy,
|
||||
} => PreparedIamMode::PluginAttemptSts {
|
||||
is_owner,
|
||||
combined_policy,
|
||||
session_policy,
|
||||
},
|
||||
_ => PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: prepared.needs_existing_object_tag,
|
||||
mode,
|
||||
};
|
||||
}
|
||||
if let Some(prepared) = state.prepared_iam_auth() {
|
||||
return prepared;
|
||||
}
|
||||
|
||||
self.prepare_builtin_auth(args).await
|
||||
}
|
||||
async fn prepare_builtin_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
|
||||
let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
@@ -1425,6 +1522,20 @@ impl<T: Store> IamSys<T> {
|
||||
/// bucket policies on an IAM denial. An error must not become an implicit deny.
|
||||
pub async fn try_eval_prepared(&self, prepared: &PreparedIamAuth, args: &Args<'_>) -> Result<bool> {
|
||||
Ok(match &prepared.mode {
|
||||
PreparedIamMode::TerminalDeny => return Err(Error::IAMActionNotAllowed),
|
||||
PreparedIamMode::PluginAttemptSts {
|
||||
is_owner,
|
||||
combined_policy,
|
||||
session_policy,
|
||||
} => {
|
||||
// A false IAM result permits bucket-policy fallback. Restricted
|
||||
// plugin credentials must instead terminate authorization.
|
||||
let session_allowed = evaluate_prepared_session_policy(session_policy, args).await == Some(true);
|
||||
if !session_allowed || !(*is_owner || combined_policy.is_allowed(args).await) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
true
|
||||
}
|
||||
PreparedIamMode::Opa => {
|
||||
let Some(opa_enable) = Self::get_policy_plugin_client().await else {
|
||||
tracing::warn!("eval_prepared: OPA mode requested but plugin is unavailable");
|
||||
@@ -2127,6 +2238,145 @@ mod tests {
|
||||
assert!(prepared.combined_policy_for_view().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn marked_signed_sts_denial_is_terminal_before_public_bucket_fallback() {
|
||||
let iam = IamSys::new(IamCache::new(StsTestMockStore::new(false)).await.unwrap());
|
||||
let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
|
||||
let signing_key = "c07-test-token-signing-key";
|
||||
let mut metadata=HashMap::from([
|
||||
("parent".into(),Value::String("sts-fallback-test-parent".into())),
|
||||
("exp".into(),serde_json::json!((OffsetDateTime::now_utc()+time::Duration::hours(1)).unix_timestamp())),
|
||||
(SESSION_POLICY_NAME.into(),Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/inside"]}]}"#))),
|
||||
]);
|
||||
metadata.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), Value::Bool(true));
|
||||
let mut marked = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
|
||||
marked.parent_user = "sts-fallback-test-parent".into();
|
||||
let marked_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&marked.session_token, signing_key)
|
||||
.expect("signed test credentials must decode")
|
||||
.claims;
|
||||
let marked_user = UserIdentity::from(marked.clone());
|
||||
iam.store
|
||||
.cache
|
||||
.add_or_update_sts_account(&marked.access_key, &marked_user, OffsetDateTime::now_utc());
|
||||
metadata.remove(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
|
||||
let mut ordinary = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
|
||||
ordinary.parent_user = marked.parent_user.clone();
|
||||
let ordinary_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&ordinary.session_token, signing_key)
|
||||
.expect("signed test credentials must decode")
|
||||
.claims;
|
||||
iam.store.cache.add_or_update_sts_account(
|
||||
&ordinary.access_key,
|
||||
&UserIdentity::from(ordinary.clone()),
|
||||
OffsetDateTime::now_utc(),
|
||||
);
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
let args = Args {
|
||||
account: &marked.access_key,
|
||||
groups: &groups,
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
bucket: "mybucket",
|
||||
object: "outside",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
claims: &marked_claims,
|
||||
deny_only: false,
|
||||
};
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
|
||||
let denied = iam.prepare_auth(&args).await;
|
||||
let disabled_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
let inside = Args {
|
||||
object: "inside",
|
||||
..args.clone()
|
||||
};
|
||||
let allowed = iam.prepare_auth(&inside).await;
|
||||
let inside_allowed = matches!(iam.try_eval_prepared(&allowed, &inside).await, Ok(true));
|
||||
let unmarked_args = Args {
|
||||
account: &ordinary.access_key,
|
||||
claims: &ordinary_claims,
|
||||
..args.clone()
|
||||
};
|
||||
let unmarked = iam.prepare_auth(&unmarked_args).await;
|
||||
let ordinary_false = matches!(iam.try_eval_prepared(&unmarked, &unmarked_args).await, Ok(false));
|
||||
let public:rustfs_policy::policy::BucketPolicy=serde_json::from_value(serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/*"]}]})).unwrap();
|
||||
let public_allows = public
|
||||
.is_allowed(&rustfs_policy::policy::BucketPolicyArgs {
|
||||
account: &marked.access_key,
|
||||
groups: &groups,
|
||||
action: args.action,
|
||||
bucket: args.bucket,
|
||||
object: args.object,
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
})
|
||||
.await;
|
||||
IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
|
||||
url: "http://127.0.0.1:8181/allow".into(),
|
||||
auth_token: String::new(),
|
||||
}))
|
||||
.await;
|
||||
let opa_marked = iam.prepare_auth(&args).await;
|
||||
let ready_terminal = matches!(iam.try_eval_prepared(&opa_marked, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
let opa_ordinary = iam.prepare_auth(&unmarked_args).await;
|
||||
let ordinary_keeps_opa = matches!(opa_ordinary.mode, PreparedIamMode::Opa);
|
||||
// The captured Disabled preparation stays on built-in intersection even
|
||||
// if an external policy client becomes ready before evaluation.
|
||||
let captured_still_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
|
||||
let pending = iam.prepare_auth(&args).await;
|
||||
let initializing_terminal = matches!(iam.try_eval_prepared(&pending, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Failed;
|
||||
let failed = iam.prepare_auth(&args).await;
|
||||
let failed_terminal = matches!(iam.try_eval_prepared(&failed, &args).await, Err(Error::IAMActionNotAllowed));
|
||||
*get_policy_plugin_state().write().await = previous;
|
||||
assert!(public_allows, "C07_PUBLIC_BUCKET_CONTROL: real public policy permits outside object");
|
||||
assert!(
|
||||
disabled_terminal && captured_still_terminal,
|
||||
"C07_SCOPE_TERMINAL_BOUND: bucket policy must not rescue restricted scope denial"
|
||||
);
|
||||
assert!(
|
||||
ready_terminal && initializing_terminal && failed_terminal,
|
||||
"C07_EXTERNAL_POLICY_BOUND: marked persisted STS cannot enter OPA"
|
||||
);
|
||||
assert!(
|
||||
inside_allowed && ordinary_false && ordinary_keeps_opa,
|
||||
"C07_UNMARKED_COMPAT: preserve ordinary STS and valid marked scope"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn restricted_vending_rejects_actual_external_policy_plugin_state() {
|
||||
let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
|
||||
IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
|
||||
url: "http://127.0.0.1:8181/v1/data/rustfs/authz/allow".into(),
|
||||
auth_token: String::new(),
|
||||
}))
|
||||
.await;
|
||||
let active_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
|
||||
.await
|
||||
.is_err();
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
|
||||
let initializing_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
|
||||
.await
|
||||
.is_err();
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Failed;
|
||||
let failed_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
|
||||
.await
|
||||
.is_err();
|
||||
*get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
|
||||
let disabled_allowed = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
|
||||
.await
|
||||
.is_ok();
|
||||
*get_policy_plugin_state().write().await = previous;
|
||||
assert!(
|
||||
active_rejected && initializing_rejected && failed_rejected,
|
||||
"C07_EXTERNAL_POLICY_BOUND: external authorization cannot bypass minted session scope"
|
||||
);
|
||||
assert!(disabled_allowed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_prepare_auth_requests_existing_object_tags_in_opa_mode() {
|
||||
@@ -2667,11 +2917,7 @@ mod tests {
|
||||
name: None,
|
||||
description: None,
|
||||
};
|
||||
let parent_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: creds,
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let parent_identity = UserIdentity::new(creds);
|
||||
let mut users = HashMap::new();
|
||||
users.insert(PARENT_USER.to_string(), parent_identity);
|
||||
|
||||
@@ -2703,11 +2949,7 @@ mod tests {
|
||||
name: None,
|
||||
description: None,
|
||||
};
|
||||
let parent_identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: creds,
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let parent_identity = UserIdentity::new(creds);
|
||||
let mut users = HashMap::new();
|
||||
users.insert(PARENT_USER.to_string(), parent_identity);
|
||||
|
||||
@@ -2820,16 +3062,12 @@ mod tests {
|
||||
async fn group_writes_stamped_before_the_cache_load_time_still_publish() {
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let member = "group-stamp-member";
|
||||
let identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: Credentials {
|
||||
access_key: member.to_string(),
|
||||
secret_key: "longenoughsecret".to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
let identity = UserIdentity::new(Credentials {
|
||||
access_key: member.to_string(),
|
||||
secret_key: "longenoughsecret".to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
iam_sys.store.cache.with_write_lock(|cache| {
|
||||
cache.add_or_update_user(member, &identity, OffsetDateTime::now_utc());
|
||||
// The startup load publishes every entity with the load time.
|
||||
|
||||
@@ -22,7 +22,11 @@
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::migration::try_migrate_iam_config;
|
||||
pub(crate) use rustfs_ecstore::api::config::com::save_config;
|
||||
pub(crate) use rustfs_ecstore::api::global::set_object_store_resolver;
|
||||
pub(crate) use rustfs_ecstore::api::layout::SetupType;
|
||||
pub(crate) use rustfs_ecstore::api::runtime::{InstanceContext, first_cluster_node_is_local, object_store_handle};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore;
|
||||
|
||||
// `update_erasure_type` is a write-side global facade entry. Its use is
|
||||
// restricted to reviewed storage_api boundaries; this test-only module is
|
||||
|
||||
@@ -33,12 +33,21 @@
|
||||
|
||||
mod ecstore_test_compat;
|
||||
|
||||
use ecstore_test_compat::fixture::{SetupType, update_erasure_type};
|
||||
use ecstore_test_compat::fixture::{
|
||||
ECStore, InstanceContext, SetupType, first_cluster_node_is_local, object_store_handle, save_config,
|
||||
set_object_store_resolver, update_erasure_type,
|
||||
};
|
||||
use rustfs_credentials::{Credentials, IAM_POLICY_CLAIM_NAME_SA};
|
||||
use rustfs_iam::cache::Cache;
|
||||
use rustfs_iam::store::object::ObjectStore;
|
||||
use rustfs_iam::store::{GroupInfo, Store};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::manager::{IamCache, IamState};
|
||||
use rustfs_iam::store::object::{IAM_CONFIG_POLICIES_PREFIX, IAM_CONFIG_USERS_PREFIX, ObjectStore};
|
||||
use rustfs_iam::store::{GroupInfo, Store, UserType};
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64};
|
||||
use std::sync::{Arc, Mutex, OnceLock, Weak};
|
||||
|
||||
const TEST_GROUP: &str = "seq-restart-group";
|
||||
const TEST_MEMBERS: [&str; 2] = ["alice", "bob"];
|
||||
@@ -69,7 +78,8 @@ async fn load_all_bypasses_namespace_lock_quorum() {
|
||||
.build()
|
||||
.await
|
||||
.ecstore;
|
||||
let store = ObjectStore::new(ecstore);
|
||||
let _route_guard = TestStoreRouteGuard::new(&ecstore);
|
||||
let store = ObjectStore::new(ecstore.clone());
|
||||
|
||||
// Seed IAM data while namespace locks still work (single-node mode).
|
||||
store
|
||||
@@ -128,3 +138,447 @@ async fn load_all_bypasses_namespace_lock_quorum() {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
const IDENTITY_PARENT: &str = "fixture-parent";
|
||||
const IDENTITY_CHILD: &str = "fixture-service-child";
|
||||
|
||||
static TEST_STORE_ROUTE: Mutex<Option<Weak<ECStore>>> = Mutex::new(None);
|
||||
static TEST_STORE_RESOLVER: OnceLock<()> = OnceLock::new();
|
||||
|
||||
struct TestStoreRouteGuard;
|
||||
|
||||
impl TestStoreRouteGuard {
|
||||
fn new(ecstore: &Arc<ECStore>) -> Self {
|
||||
TEST_STORE_RESOLVER.get_or_init(|| {
|
||||
assert!(
|
||||
set_object_store_resolver(Arc::new(|| {
|
||||
TEST_STORE_ROUTE
|
||||
.lock()
|
||||
.expect("lock test store route")
|
||||
.as_ref()
|
||||
.and_then(Weak::upgrade)
|
||||
})),
|
||||
"test binary must own its resolver registration"
|
||||
);
|
||||
});
|
||||
let mut route = TEST_STORE_ROUTE.lock().expect("lock test store route");
|
||||
assert!(route.is_none(), "another test store route is active");
|
||||
*route = Some(Arc::downgrade(ecstore));
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestStoreRouteGuard {
|
||||
fn drop(&mut self) {
|
||||
*TEST_STORE_ROUTE.lock().expect("lock test store route for cleanup") = None;
|
||||
}
|
||||
}
|
||||
|
||||
struct IdentityStoreFixture {
|
||||
previous_store: Option<Arc<ECStore>>,
|
||||
ecstore: Arc<ECStore>,
|
||||
manager: IamCache<ObjectStore>,
|
||||
_route_guard: TestStoreRouteGuard,
|
||||
_temp_dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl IdentityStoreFixture {
|
||||
async fn new() -> Self {
|
||||
let temp_dir = tempfile::TempDir::with_prefix("rustfs_iam_identity_fixture_").expect("create identity fixture directory");
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp_dir.path())
|
||||
.init_bucket_metadata(false)
|
||||
.instance_ctx(ctx.clone())
|
||||
.build()
|
||||
.await;
|
||||
ctx.set_endpoints(env.endpoint_pools.clone());
|
||||
let ecstore = env.ecstore;
|
||||
let previous_store = object_store_handle();
|
||||
let route_guard = TestStoreRouteGuard::new(&ecstore);
|
||||
let selected = object_store_handle().expect("fixture route selects its ECStore");
|
||||
assert!(Arc::ptr_eq(&selected, &ecstore), "fixture route must select the isolated ECStore");
|
||||
drop(selected);
|
||||
assert!(first_cluster_node_is_local().await, "isolated fixture has a local first endpoint");
|
||||
temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], rustfs_iam::build_iam_sys(ecstore.clone()))
|
||||
.await
|
||||
.expect("initialize IAM metadata through production owner");
|
||||
let store = ObjectStore::new(ecstore.clone());
|
||||
let parent = UserIdentity::new(Credentials {
|
||||
access_key: IDENTITY_PARENT.to_string(),
|
||||
secret_key: "fixture-parent-secret".to_string(),
|
||||
expiration: None,
|
||||
session_token: String::new(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
let child = UserIdentity::new(Credentials {
|
||||
access_key: IDENTITY_CHILD.to_string(),
|
||||
secret_key: "fixture-child-secret".to_string(),
|
||||
expiration: None,
|
||||
session_token: String::new(),
|
||||
status: "on".to_string(),
|
||||
parent_user: IDENTITY_PARENT.to_string(),
|
||||
claims: Some(HashMap::from([(
|
||||
IAM_POLICY_CLAIM_NAME_SA.to_string(),
|
||||
serde_json::json!("inherited-policy"),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(child.credentials.is_service_account(), "child fixture must be a service account");
|
||||
store
|
||||
.save_user_identity(IDENTITY_PARENT, UserType::Reg, parent, None)
|
||||
.await
|
||||
.expect("persist parent identity");
|
||||
store
|
||||
.save_user_identity(IDENTITY_CHILD, UserType::Svc, child, None)
|
||||
.await
|
||||
.expect("persist service child identity");
|
||||
let manager = IamCache {
|
||||
cache: Cache::default(),
|
||||
api: store,
|
||||
state: Arc::new(AtomicU8::new(IamState::Ready as u8)),
|
||||
loading: Arc::new(AtomicBool::new(false)),
|
||||
roles: HashMap::new(),
|
||||
send_chan: tokio::sync::mpsc::channel::<i64>(1).0,
|
||||
last_timestamp: AtomicI64::new(0),
|
||||
sync_failures: AtomicU64::new(0),
|
||||
sync_successes: AtomicU64::new(0),
|
||||
last_sync_duration_millis: AtomicU64::new(0),
|
||||
};
|
||||
manager
|
||||
.load_user(IDENTITY_PARENT)
|
||||
.await
|
||||
.expect("load persisted parent into cache");
|
||||
manager
|
||||
.load_user(IDENTITY_CHILD)
|
||||
.await
|
||||
.expect("load persisted service child into cache");
|
||||
Self {
|
||||
previous_store,
|
||||
ecstore,
|
||||
manager,
|
||||
_route_guard: route_guard,
|
||||
_temp_dir: temp_dir,
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_parent_and_child(&self) {
|
||||
let parent = self
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect("read parent from real storage");
|
||||
assert_eq!(parent.credentials.access_key, IDENTITY_PARENT);
|
||||
assert_eq!(parent.credentials.parent_user, "");
|
||||
let child = self
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_CHILD, UserType::Svc)
|
||||
.await
|
||||
.expect("read child from real storage");
|
||||
assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
|
||||
assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
|
||||
assert!(child.credentials.is_service_account());
|
||||
let cached_parent = self.manager.get_user(IDENTITY_PARENT).await.expect("parent remains in cache");
|
||||
assert_eq!(cached_parent.credentials.access_key, IDENTITY_PARENT);
|
||||
assert_eq!(cached_parent.credentials.parent_user, "");
|
||||
let cached_child = self
|
||||
.manager
|
||||
.get_user(IDENTITY_CHILD)
|
||||
.await
|
||||
.expect("service child remains in cache");
|
||||
assert_eq!(cached_child.credentials.access_key, IDENTITY_CHILD);
|
||||
assert_eq!(cached_child.credentials.parent_user, IDENTITY_PARENT);
|
||||
assert!(cached_child.credentials.is_service_account());
|
||||
}
|
||||
|
||||
async fn overwrite_parent_bytes(&self, bytes: &[u8]) {
|
||||
let path = format!("{}{IDENTITY_PARENT}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
|
||||
save_config(self.ecstore.clone(), &path, bytes.to_vec())
|
||||
.await
|
||||
.expect("overwrite real parent identity bytes");
|
||||
}
|
||||
|
||||
async fn assert_child_storage_and_identity_cache(&self) {
|
||||
let child = self
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_CHILD, UserType::Svc)
|
||||
.await
|
||||
.expect("notification must preserve persisted service child");
|
||||
assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
|
||||
assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
|
||||
assert!(child.credentials.is_service_account());
|
||||
let parent = self
|
||||
.manager
|
||||
.get_user(IDENTITY_PARENT)
|
||||
.await
|
||||
.expect("failed notification preserves cached parent");
|
||||
assert_eq!(parent.credentials.access_key, IDENTITY_PARENT);
|
||||
assert_eq!(parent.credentials.parent_user, "");
|
||||
let child = self
|
||||
.manager
|
||||
.get_user(IDENTITY_CHILD)
|
||||
.await
|
||||
.expect("failed notification preserves cached child");
|
||||
assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
|
||||
assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
|
||||
assert!(child.credentials.is_service_account());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_store_fixture_roundtrips_parent_and_service_child() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
let before = fixture.previous_store.clone();
|
||||
let selected = object_store_handle().expect("fixture route selects its ECStore");
|
||||
assert!(Arc::ptr_eq(&selected, &fixture.ecstore));
|
||||
drop(selected);
|
||||
fixture.assert_parent_and_child().await;
|
||||
drop(fixture);
|
||||
assert!(TEST_STORE_ROUTE.lock().expect("inspect cleared test store route").is_none());
|
||||
match (before, object_store_handle()) {
|
||||
(None, None) => {}
|
||||
(Some(before), Some(after)) => assert!(Arc::ptr_eq(&before, &after), "fixture drop restores prior store identity"),
|
||||
_ => panic!("fixture drop must restore the prior store option"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_notification_valid_parent_preserves_child_storage_and_cache() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.assert_parent_and_child().await;
|
||||
fixture
|
||||
.manager
|
||||
.user_notification_handler(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect("refresh valid parent notification");
|
||||
fixture.assert_parent_and_child().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_store_fixture_raw_valid_overwrite_is_observed() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
let parent = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect("load valid parent before raw overwrite");
|
||||
let mut raw = serde_json::to_value(parent).expect("serialize valid parent fixture");
|
||||
raw["credentials"]["name"] = serde_json::json!("updated-name");
|
||||
let path = format!("{}{IDENTITY_PARENT}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
|
||||
save_config(
|
||||
fixture.ecstore.clone(),
|
||||
&path,
|
||||
serde_json::to_vec(&raw).expect("encode raw parent fixture"),
|
||||
)
|
||||
.await
|
||||
.expect("overwrite parent through real config storage");
|
||||
fixture
|
||||
.manager
|
||||
.load_user(IDENTITY_PARENT)
|
||||
.await
|
||||
.expect("reload parent after raw overwrite");
|
||||
let stored = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect("read overwritten parent from storage");
|
||||
assert_eq!(stored.credentials.name.as_deref(), Some("updated-name"));
|
||||
let cached = fixture
|
||||
.manager
|
||||
.get_user(IDENTITY_PARENT)
|
||||
.await
|
||||
.expect("read overwritten parent from cache");
|
||||
assert_eq!(cached.credentials.name.as_deref(), Some("updated-name"));
|
||||
fixture.assert_parent_and_child().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_corrupt_reads_preserve_crypto_error() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.overwrite_parent_bytes(&[0xff, 0xfe, 0xfd]).await;
|
||||
let locked = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect_err("corrupt stored identity must fail locked read");
|
||||
assert!(matches!(locked, IamError::CryptoError(_)), "preserve crypto error, got {locked:?}");
|
||||
let mut users = HashMap::new();
|
||||
let no_lock = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
|
||||
.await
|
||||
.expect_err("corrupt stored identity must fail no-lock read");
|
||||
assert!(matches!(no_lock, IamError::CryptoError(_)), "preserve crypto error, got {no_lock:?}");
|
||||
assert!(users.is_empty(), "failed no-lock read must not populate users");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_corrupt_notification_preserves_child_storage_and_cache() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.assert_parent_and_child().await;
|
||||
fixture.overwrite_parent_bytes(&[0xff, 0xfe, 0xfd]).await;
|
||||
let notification = fixture
|
||||
.manager
|
||||
.user_notification_handler(IDENTITY_PARENT, UserType::Reg)
|
||||
.await;
|
||||
fixture.assert_child_storage_and_identity_cache().await;
|
||||
let error = notification.expect_err("corrupt parent notification must return an error");
|
||||
assert!(
|
||||
matches!(error, IamError::CryptoError(_)),
|
||||
"preserve notification crypto error, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_invalid_schema_preserves_decode_error() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.assert_parent_and_child().await;
|
||||
fixture
|
||||
.overwrite_parent_bytes(br#"{"version":"not-an-integer","credentials":{}}"#)
|
||||
.await;
|
||||
let error = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect_err("invalid identity field type must fail decode");
|
||||
assert_identity_data_error(error);
|
||||
let mut users = HashMap::new();
|
||||
let error = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
|
||||
.await
|
||||
.expect_err("invalid identity field type must fail no-lock decode");
|
||||
assert_identity_data_error(error);
|
||||
assert!(users.is_empty(), "failed decode must not populate users");
|
||||
let notification = fixture
|
||||
.manager
|
||||
.user_notification_handler(IDENTITY_PARENT, UserType::Reg)
|
||||
.await;
|
||||
fixture.assert_child_storage_and_identity_cache().await;
|
||||
assert_identity_data_error(notification.expect_err("invalid parent notification must return decode error"));
|
||||
}
|
||||
|
||||
fn assert_identity_data_error(error: IamError) {
|
||||
let IamError::Io(inner) = error else {
|
||||
panic!("identity decode must preserve Io serde error, got {error:?}");
|
||||
};
|
||||
let cause = inner
|
||||
.get_ref()
|
||||
.and_then(|cause| cause.downcast_ref::<serde_json::Error>())
|
||||
.expect("identity decode error retains serde source");
|
||||
assert!(cause.is_data(), "identity field type must produce serde Data error");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_physically_missing_is_no_such_user() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
let name = "physically-missing-identity";
|
||||
let error = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(name, UserType::Reg)
|
||||
.await
|
||||
.expect_err("missing identity must be absent");
|
||||
assert!(
|
||||
matches!(error, IamError::NoSuchUser(ref missing) if missing == name),
|
||||
"preserve absent name, got {error:?}"
|
||||
);
|
||||
let mut users = HashMap::new();
|
||||
let error = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_no_lock(name, UserType::Reg, &mut users)
|
||||
.await
|
||||
.expect_err("missing no-lock identity must be absent");
|
||||
assert!(
|
||||
matches!(error, IamError::NoSuchUser(ref missing) if missing == name),
|
||||
"preserve absent name, got {error:?}"
|
||||
);
|
||||
assert!(users.is_empty(), "missing read must not populate users");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_empty_reads_preserve_crypto_error() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.overwrite_parent_bytes(&[]).await;
|
||||
let locked = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_identity(IDENTITY_PARENT, UserType::Reg)
|
||||
.await
|
||||
.expect_err("empty stored identity must fail locked read");
|
||||
assert!(matches!(locked, IamError::CryptoError(_)), "preserve crypto error, got {locked:?}");
|
||||
let mut users = HashMap::new();
|
||||
let no_lock = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
|
||||
.await
|
||||
.expect_err("empty stored identity must fail no-lock read");
|
||||
assert!(matches!(no_lock, IamError::CryptoError(_)), "preserve crypto error, got {no_lock:?}");
|
||||
assert!(users.is_empty(), "failed no-lock read must not populate users");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn identity_empty_notification_preserves_child_storage_and_cache() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
fixture.assert_parent_and_child().await;
|
||||
fixture.overwrite_parent_bytes(&[]).await;
|
||||
let notification = fixture
|
||||
.manager
|
||||
.user_notification_handler(IDENTITY_PARENT, UserType::Reg)
|
||||
.await;
|
||||
fixture.assert_child_storage_and_identity_cache().await;
|
||||
let error = notification.expect_err("empty parent notification must return an error");
|
||||
assert!(
|
||||
matches!(error, IamError::CryptoError(_)),
|
||||
"preserve notification crypto error, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn policy_empty_payload_retains_no_such_policy() {
|
||||
let fixture = IdentityStoreFixture::new().await;
|
||||
let name = "empty-policy";
|
||||
let path = format!("{}{name}/policy.json", IAM_CONFIG_POLICIES_PREFIX.as_str());
|
||||
save_config(fixture.ecstore.clone(), &path, Vec::new())
|
||||
.await
|
||||
.expect("save real empty policy payload");
|
||||
let locked = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_policy(name)
|
||||
.await
|
||||
.err()
|
||||
.expect("empty policy must retain absent-policy behavior");
|
||||
assert!(matches!(locked, IamError::NoSuchPolicy), "preserve policy error, got {locked:?}");
|
||||
let mut policies = HashMap::new();
|
||||
let no_lock = fixture
|
||||
.manager
|
||||
.api
|
||||
.load_policy_doc_no_lock(name, &mut policies)
|
||||
.await
|
||||
.expect_err("empty no-lock policy must retain absent-policy behavior");
|
||||
assert!(matches!(no_lock, IamError::NoSuchPolicy), "preserve policy error, got {no_lock:?}");
|
||||
assert!(policies.is_empty(), "failed no-lock policy read must not populate policies");
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::metrics::schema::scanner::*;
|
||||
/// Scanner statistics.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ScannerStats {
|
||||
pub startup: rustfs_scanner_metrics::startup::ScannerStartupStatus,
|
||||
/// Number of bucket-drive scans finished
|
||||
pub bucket_scans_finished: u64,
|
||||
/// Number of bucket-drive scans started
|
||||
@@ -316,6 +317,23 @@ fn collect_scanner_metrics_with_runtime(stats: &ScannerStats, runtime: Option<&S
|
||||
}
|
||||
|
||||
let mut metrics = vec![
|
||||
PrometheusMetric::from_descriptor(&SCANNER_RUNTIME_ACTIVE_MD, if stats.startup.runtime_active { 1.0 } else { 0.0 }),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_STARTUP_PHASE_MD, stats.startup.phase.code() as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_STARTUP_STARTED_UNIX_SECONDS_MD, stats.startup.started_at_unix_secs as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_STARTUP_PHASE_AGE_SECONDS_MD, stats.startup.phase_age_seconds as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_STARTUP_PLANNED_ATTEMPT_UNIX_SECONDS_MD,
|
||||
stats.startup.planned_attempt_at_unix_secs.unwrap_or_default() as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_STARTUP_WAIT_REMAINING_SECONDS_MD,
|
||||
stats.startup.wait_remaining_seconds as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_STARTUP_WAIT_OVERDUE_SECONDS_MD, stats.startup.wait_overdue_seconds as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_STARTUP_FIRST_ATTEMPT_UNIX_SECONDS_MD,
|
||||
stats.startup.first_attempt_at_unix_secs.unwrap_or_default() as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64),
|
||||
@@ -661,6 +679,7 @@ mod tests {
|
||||
age_seconds: 7,
|
||||
}],
|
||||
stats: ScannerStats {
|
||||
startup: Default::default(),
|
||||
bucket_scans_finished: 100,
|
||||
bucket_scans_started: 100,
|
||||
bucket_scans_failed: 2,
|
||||
@@ -749,7 +768,7 @@ mod tests {
|
||||
let metrics = collect_scanner_runtime_metrics(&stats);
|
||||
report_metrics(&metrics);
|
||||
|
||||
assert_eq!(metrics.len(), 105);
|
||||
assert_eq!(metrics.len(), 113);
|
||||
|
||||
let objects = metrics.iter().find(|m| m.value == 1000000.0);
|
||||
assert!(objects.is_some());
|
||||
@@ -1163,12 +1182,48 @@ mod tests {
|
||||
assert_eq!(current_usage_executed.map(|m| m.value), Some(8.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_metrics_use_one_bounded_snapshot() {
|
||||
use rustfs_scanner_metrics::startup::{ScannerStartupPhase, ScannerStartupStatus};
|
||||
let stats = ScannerStats {
|
||||
startup: ScannerStartupStatus {
|
||||
runtime_active: true,
|
||||
phase: ScannerStartupPhase::Waiting,
|
||||
started_at_unix_secs: 1_700_000_000,
|
||||
phase_age_seconds: 12,
|
||||
planned_attempt_at_unix_secs: Some(1_700_000_120),
|
||||
wait_remaining_seconds: 108,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
for (descriptor, expected) in [
|
||||
(&*SCANNER_RUNTIME_ACTIVE_MD, 1.0),
|
||||
(&*SCANNER_STARTUP_PHASE_MD, 3.0),
|
||||
(&*SCANNER_STARTUP_STARTED_UNIX_SECONDS_MD, 1_700_000_000.0),
|
||||
(&*SCANNER_STARTUP_PHASE_AGE_SECONDS_MD, 12.0),
|
||||
(&*SCANNER_STARTUP_PLANNED_ATTEMPT_UNIX_SECONDS_MD, 1_700_000_120.0),
|
||||
(&*SCANNER_STARTUP_WAIT_REMAINING_SECONDS_MD, 108.0),
|
||||
(&*SCANNER_STARTUP_WAIT_OVERDUE_SECONDS_MD, 0.0),
|
||||
(&*SCANNER_STARTUP_FIRST_ATTEMPT_UNIX_SECONDS_MD, 0.0),
|
||||
] {
|
||||
let selected: Vec<_> = metrics
|
||||
.iter()
|
||||
.filter(|metric| metric.name == descriptor.get_full_metric_name())
|
||||
.collect();
|
||||
assert_eq!(selected.len(), 1, "one producer per startup gauge");
|
||||
assert_eq!(selected[0].value, expected);
|
||||
assert!(selected[0].labels.is_empty(), "startup gauges must not add per-object labels");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_scanner_metrics_default() {
|
||||
let stats = ScannerStats::default();
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 82);
|
||||
assert_eq!(metrics.len(), 90);
|
||||
for metric in &metrics {
|
||||
assert_eq!(metric.value, 0.0);
|
||||
if metric.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name() {
|
||||
|
||||
@@ -787,3 +787,75 @@ pub static SCANNER_PARTIAL_CYCLES_BY_REASON_MD: LazyLock<MetricDescriptor> = Laz
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_RUNTIME_ACTIVE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("runtime_active".to_string()),
|
||||
"Whether the latest local scanner runtime owns its lifecycle guard, not a health proof",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_STARTUP_PHASE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("startup_phase".to_string()),
|
||||
"Local scanner startup phase: 0 unknown, 1 initializing, 2 usage inspection, 3 waiting, 4 attempted, 5 stopped",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_STARTUP_STARTED_UNIX_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("startup_started_unix_seconds".to_string()),
|
||||
"Unix timestamp when the latest local scanner runtime was initialized, or zero if unknown",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_STARTUP_PHASE_AGE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("startup_phase_age_seconds".to_string()),
|
||||
"Seconds since the latest local scanner startup transition",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_STARTUP_PLANNED_ATTEMPT_UNIX_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("startup_planned_attempt_unix_seconds".to_string()),
|
||||
"Estimated Unix timestamp of the sampled startup attempt deadline, not scan completion; zero if unavailable",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_STARTUP_WAIT_REMAINING_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("startup_wait_remaining_seconds".to_string()),
|
||||
"Monotonic seconds remaining before the sampled startup attempt deadline",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_STARTUP_WAIT_OVERDUE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("startup_wait_overdue_seconds".to_string()),
|
||||
"Monotonic seconds past the sampled startup deadline while the runtime remains in startup waiting",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_STARTUP_FIRST_ATTEMPT_UNIX_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("startup_first_attempt_unix_seconds".to_string()),
|
||||
"Unix timestamp of first normal scanner leadership/recovery attempt after startup recovery, not scan success; zero if unobserved",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -1652,6 +1652,7 @@ pub(crate) async fn collect_scanner_runtime_metric_stats() -> Option<ScannerRunt
|
||||
last_cycle_bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.last_cycle_bucket_drive_results),
|
||||
active_bucket_drive_scans: scanner_active_bucket_drive_stats(&runtime_details.active_bucket_drive_scans),
|
||||
stats: ScannerStats {
|
||||
startup: metrics.startup,
|
||||
bucket_scans_finished,
|
||||
bucket_scans_started,
|
||||
bucket_scans_failed,
|
||||
@@ -1777,6 +1778,29 @@ pub async fn collect_compression_cluster_stats() -> Option<CompressionClusterSta
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[tokio::test]
|
||||
async fn scanner_startup_snapshot_reaches_the_real_stats_mapping() {
|
||||
use rustfs_scanner_metrics::startup::ScannerStartupPhase;
|
||||
let metrics = rustfs_scanner_metrics::metrics::global_metrics();
|
||||
let token = metrics.startup().begin();
|
||||
metrics
|
||||
.startup()
|
||||
.record_wait(token, tokio::time::Instant::now() + std::time::Duration::from_secs(300));
|
||||
let mapped = super::collect_scanner_runtime_metric_stats().await.expect("scanner stats");
|
||||
assert_eq!(mapped.stats.startup.phase, ScannerStartupPhase::Waiting);
|
||||
assert!(mapped.stats.startup.runtime_active);
|
||||
assert!(mapped.stats.startup.planned_attempt_at_unix_secs.is_some());
|
||||
assert!(mapped.stats.startup.wait_remaining_seconds > 0);
|
||||
assert!(mapped.stats.startup.first_attempt_at_unix_secs.is_none());
|
||||
metrics.startup().finish(token);
|
||||
let stopped = super::collect_scanner_runtime_metric_stats()
|
||||
.await
|
||||
.expect("stopped scanner stats");
|
||||
assert_eq!(stopped.stats.startup.phase, ScannerStartupPhase::Stopped);
|
||||
assert!(!stopped.stats.startup.runtime_active);
|
||||
assert!(stopped.stats.startup.planned_attempt_at_unix_secs.is_none());
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot;
|
||||
use std::io::{Read, Write};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user